notebooks/gpt2/multi_linear_probe.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e4bb8cbd",
   "metadata": {
    "papermill": {
     "duration": 0.003707,
     "end_time": "2024-03-23T22:58:24.695263",
     "exception": false,
     "start_time": "2024-03-23T22:58:24.691556",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Train multiple linear probes at once\n",
    "In this notebook we will find out at which layer a transformer has the most linearly seperable information required to do causal language modelling on wikitext data. The nice thing about *transformer_heads* is that this will all be possible with just one training run."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82e27799",
   "metadata": {},
   "source": [
    "**GPU Requirements:** For running with GPT-2 you may be fine with just 8GB of GPU RAM. With about 24GB you should be able to run any 7B model (E.g. Llama or Mistral). With 80GB (A100) GPU you may be able to run a 70B model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "681731a4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T22:58:24.703080Z",
     "iopub.status.busy": "2024-03-23T22:58:24.702670Z",
     "iopub.status.idle": "2024-03-23T22:58:31.820350Z",
     "shell.execute_reply": "2024-03-23T22:58:31.819809Z"
    },
    "papermill": {
     "duration": 7.123205,
     "end_time": "2024-03-23T22:58:31.821921",
     "exception": false,
     "start_time": "2024-03-23T22:58:24.698716",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformer_heads import load_headed\n",
    "from datasets import load_dataset\n",
    "from transformers import (\n",
    "    AutoTokenizer,\n",
    "    MistralForCausalLM,\n",
    "    Trainer,\n",
    "    BitsAndBytesConfig,\n",
    "    TrainingArguments,\n",
    "    GPT2Model,\n",
    "    GPT2LMHeadModel,\n",
    ")\n",
    "from transformer_heads.util.helpers import DataCollatorWithPadding, get_model_params\n",
    "from peft import LoraConfig\n",
    "from transformer_heads.config import HeadConfig\n",
    "from transformer_heads.util.model import print_trainable_parameters\n",
    "from transformer_heads.util.evaluate import evaluate_head_wise, get_top_n_preds\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "77f786cd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T22:58:31.831209Z",
     "iopub.status.busy": "2024-03-23T22:58:31.830854Z",
     "iopub.status.idle": "2024-03-23T22:58:31.833725Z",
     "shell.execute_reply": "2024-03-23T22:58:31.833289Z"
    },
    "papermill": {
     "duration": 0.008139,
     "end_time": "2024-03-23T22:58:31.835006",
     "exception": false,
     "start_time": "2024-03-23T22:58:31.826867",
     "status": "completed"
    },
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# GPT2 is the fastest and requires fewest memory. However, this works just the same with any Llama or Mistral model. Just change model_path to its huggingface path.\n",
    "model_path = \"gpt2\"\n",
    "train_epochs = 1\n",
    "eval_epochs = 1\n",
    "logging_steps = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5ac9fe1a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T22:58:31.853948Z",
     "iopub.status.busy": "2024-03-23T22:58:31.853653Z",
     "iopub.status.idle": "2024-03-23T22:58:32.281560Z",
     "shell.execute_reply": "2024-03-23T22:58:32.281082Z"
    },
    "papermill": {
     "duration": 0.433529,
     "end_time": "2024-03-23T22:58:32.282828",
     "exception": false,
     "start_time": "2024-03-23T22:58:31.849299",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'vocab_size': 50257, 'n_positions': 1024, 'n_embd': 768, 'n_layer': 12, 'n_head': 12, 'n_inner': None, 'activation_function': 'gelu_new', 'resid_pdrop': 0.1, 'embd_pdrop': 0.1, 'attn_pdrop': 0.1, 'layer_norm_epsilon': 1e-05, 'initializer_range': 0.02, 'summary_type': 'cls_index', 'summary_use_proj': True, 'summary_activation': None, 'summary_first_dropout': 0.1, 'summary_proj_to_labels': True, 'scale_attn_weights': True, 'use_cache': True, 'scale_attn_by_inverse_layer_idx': False, 'reorder_and_upcast_attn': False, 'bos_token_id': 50256, 'eos_token_id': 50256, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': None, 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['GPT2LMHeadModel'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'pad_token_id': None, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': {'text-generation': {'do_sample': True, 'max_length': 50}}, 'problem_type': None, '_name_or_path': 'gpt2', 'transformers_version': '4.37.2', 'model_type': 'gpt2', 'n_ctx': 1024, 'model_class': <class 'transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel'>, 'hidden_size': 768}\n"
     ]
    }
   ],
   "source": [
    "model_params = get_model_params(model_path)\n",
    "model_class = model_params[\"model_class\"]\n",
    "hidden_size = model_params[\"hidden_size\"]\n",
    "vocab_size = model_params[\"vocab_size\"]\n",
    "print(model_params)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35ce02a4",
   "metadata": {
    "papermill": {
     "duration": 0.003281,
     "end_time": "2024-03-23T22:58:32.289653",
     "exception": false,
     "start_time": "2024-03-23T22:58:32.286372",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Let's define a lot of heads in a loop. The heads will be hooked at layer -1,-3,-5,-7,-9,-11,-13. This is using python indexing: Layer -1 means after the last transformer block for example. We'll keep the original pretrained lm_head of the transformer model for comparison."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6f828a25",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T22:58:32.296984Z",
     "iopub.status.busy": "2024-03-23T22:58:32.296748Z",
     "iopub.status.idle": "2024-03-23T22:58:32.301285Z",
     "shell.execute_reply": "2024-03-23T22:58:32.300824Z"
    },
    "papermill": {
     "duration": 0.009685,
     "end_time": "2024-03-23T22:58:32.302543",
     "exception": false,
     "start_time": "2024-03-23T22:58:32.292858",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "head_configs = [\n",
    "    HeadConfig(\n",
    "        name=f\"wikitext_head_{(1+(i-1)*2)}\",\n",
    "        layer_hook=-(1 + (i - 1) * 2),\n",
    "        in_size=hidden_size,\n",
    "        hidden_size=0,\n",
    "        num_layers=1,\n",
    "        output_activation=\"linear\",\n",
    "        is_causal_lm=True,\n",
    "        loss_fct=\"cross_entropy\",\n",
    "        num_outputs=vocab_size,\n",
    "        is_regression=False,\n",
    "        output_bias=False,\n",
    "    )\n",
    "    for i in range(1, 8)\n",
    "]\n",
    "head_configs.append(\n",
    "    HeadConfig(\n",
    "        name=f\"lm_head\",\n",
    "        layer_hook=-1,\n",
    "        in_size=hidden_size,\n",
    "        hidden_size=0,\n",
    "        num_layers=1,\n",
    "        output_activation=\"linear\",\n",
    "        is_causal_lm=True,\n",
    "        loss_fct=\"cross_entropy\",\n",
    "        num_outputs=vocab_size,\n",
    "        is_regression=False,\n",
    "        output_bias=False,\n",
    "        trainable=False,\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7e30ef50",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T22:58:32.310196Z",
     "iopub.status.busy": "2024-03-23T22:58:32.309821Z",
     "iopub.status.idle": "2024-03-23T22:58:40.118916Z",
     "shell.execute_reply": "2024-03-23T22:58:40.118313Z"
    },
    "papermill": {
     "duration": 7.814439,
     "end_time": "2024-03-23T22:58:40.120379",
     "exception": false,
     "start_time": "2024-03-23T22:58:32.305940",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "dd = load_dataset(\"wikitext\", \"wikitext-2-v1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acb0d635",
   "metadata": {
    "papermill": {
     "duration": 0.003378,
     "end_time": "2024-03-23T22:58:40.127580",
     "exception": false,
     "start_time": "2024-03-23T22:58:40.124202",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "In the *tokenize_function*, we define labels for each head. For causal_lm, this is just the copied input_ids."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "33282ceb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T22:58:40.135346Z",
     "iopub.status.busy": "2024-03-23T22:58:40.135007Z",
     "iopub.status.idle": "2024-03-23T22:58:45.235941Z",
     "shell.execute_reply": "2024-03-23T22:58:45.235411Z"
    },
    "papermill": {
     "duration": 5.106091,
     "end_time": "2024-03-23T22:58:45.237029",
     "exception": false,
     "start_time": "2024-03-23T22:58:40.130938",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "15b3bce62ac54e59afcf63cbeb39ccc8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/2870 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1aa2efc61fea4ea9910d8075ce5a9f4f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/23627 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b975852c5574cc1b819e8b07a9c467e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/2460 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(model_path)\n",
    "if tokenizer.pad_token_id is None:\n",
    "    tokenizer.pad_token = tokenizer.eos_token\n",
    "\n",
    "\n",
    "def tokenize_function(examples):\n",
    "    out = tokenizer(examples[\"text\"], padding=False, truncation=True)\n",
    "    for hc in head_configs:\n",
    "        out[hc.name] = out[\"input_ids\"].copy()\n",
    "    return out\n",
    "\n",
    "\n",
    "for split in dd.keys():\n",
    "    dd[split] = dd[split].filter(function=lambda example: len(example[\"text\"]) > 10)\n",
    "    dd[split] = dd[split].map(tokenize_function, batched=True)\n",
    "dd.set_format(\n",
    "    type=\"torch\",\n",
    "    columns=[\"input_ids\", \"attention_mask\"] + [x.name for x in head_configs],\n",
    ")\n",
    "for split in dd.keys():\n",
    "    dd[split] = dd[split].remove_columns(\"text\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e80de022",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T22:58:45.247002Z",
     "iopub.status.busy": "2024-03-23T22:58:45.246735Z",
     "iopub.status.idle": "2024-03-23T22:58:58.504487Z",
     "shell.execute_reply": "2024-03-23T22:58:58.503961Z"
    },
    "papermill": {
     "duration": 13.263815,
     "end_time": "2024-03-23T22:58:58.505756",
     "exception": false,
     "start_time": "2024-03-23T22:58:45.241941",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of TransformerWithHeads were not initialized from the model checkpoint at gpt2 and are newly initialized: ['heads.wikitext_head_1.lins.0.weight', 'heads.wikitext_head_11.lins.0.weight', 'heads.wikitext_head_13.lins.0.weight', 'heads.wikitext_head_3.lins.0.weight', 'heads.wikitext_head_5.lins.0.weight', 'heads.wikitext_head_7.lins.0.weight', 'heads.wikitext_head_9.lins.0.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "quantization_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=True,\n",
    "    load_in_8bit=False,\n",
    "    llm_int8_threshold=6.0,\n",
    "    llm_int8_has_fp16_weight=False,\n",
    "    bnb_4bit_compute_dtype=torch.float32,\n",
    "    bnb_4bit_use_double_quant=True,\n",
    "    bnb_4bit_quant_type=\"nf4\",\n",
    ")\n",
    "\n",
    "model = load_headed(\n",
    "    model_class,\n",
    "    model_path,\n",
    "    head_configs=head_configs,\n",
    "    quantization_config=quantization_config,\n",
    "    device_map={\"\": torch.cuda.current_device()},\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4630a4bc",
   "metadata": {
    "papermill": {
     "duration": 0.003845,
     "end_time": "2024-03-23T22:58:58.514492",
     "exception": false,
     "start_time": "2024-03-23T22:58:58.510647",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Huggingface tells us that our newly added heads are newly initialized. Great."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1d0c5d7e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T22:58:58.523318Z",
     "iopub.status.busy": "2024-03-23T22:58:58.522913Z",
     "iopub.status.idle": "2024-03-23T22:58:58.526708Z",
     "shell.execute_reply": "2024-03-23T22:58:58.526224Z"
    },
    "papermill": {
     "duration": 0.009505,
     "end_time": "2024-03-23T22:58:58.527772",
     "exception": false,
     "start_time": "2024-03-23T22:58:58.518267",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all params: 352154112 || trainable params: 270181632 || trainable%: 76.72255492504372\n",
      "params by dtype: defaultdict(<class 'int'>, {torch.float32: 309686784, torch.uint8: 42467328})\n",
      "trainable params by dtype: defaultdict(<class 'int'>, {torch.float32: 270181632})\n"
     ]
    }
   ],
   "source": [
    "print_trainable_parameters(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b9a7478",
   "metadata": {
    "papermill": {
     "duration": 0.003827,
     "end_time": "2024-03-23T22:58:58.535511",
     "exception": false,
     "start_time": "2024-03-23T22:58:58.531684",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "A lot of heads with large vocab size -> High amount of trainable parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "205aa27a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T22:58:58.544373Z",
     "iopub.status.busy": "2024-03-23T22:58:58.544065Z",
     "iopub.status.idle": "2024-03-23T22:58:58.547699Z",
     "shell.execute_reply": "2024-03-23T22:58:58.547278Z"
    },
    "papermill": {
     "duration": 0.009398,
     "end_time": "2024-03-23T22:58:58.548758",
     "exception": false,
     "start_time": "2024-03-23T22:58:58.539360",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['input_ids', 'attention_mask', 'wikitext_head_1', 'wikitext_head_3', 'wikitext_head_5', 'wikitext_head_7', 'wikitext_head_9', 'wikitext_head_11', 'wikitext_head_13', 'lm_head'],\n",
       "    num_rows: 23627\n",
       "})"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dd[\"train\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "34af8dca",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T22:58:58.557960Z",
     "iopub.status.busy": "2024-03-23T22:58:58.557585Z",
     "iopub.status.idle": "2024-03-23T22:58:58.671110Z",
     "shell.execute_reply": "2024-03-23T22:58:58.670631Z"
    },
    "papermill": {
     "duration": 0.119365,
     "end_time": "2024-03-23T22:58:58.672241",
     "exception": false,
     "start_time": "2024-03-23T22:58:58.552876",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'wikitext_head_1': ['llan', 'genre', 'pal', ' tennis', ' shack'], 'wikitext_head_3': [' injure', 'ersion', '525', 'cluded', ' differed'], 'wikitext_head_5': [' pun', ' Request', ' Close', ' evaluation', 'chanted'], 'wikitext_head_7': [' z', ' throwing', ' wrongful', 'write', '\u899a\u9192'], 'wikitext_head_9': [' walking', ' Republic', 'Rex', ' Tur', ' gubernatorial'], 'wikitext_head_11': [' margins', ' step', 'inia', 'Prosecut', ' Finance'], 'wikitext_head_13': ['ony', ' Sher', ' Trout', ' Pas', ' unit'], 'lm_head': [' the', ' this', ' these', ' that', ' his']}\n"
     ]
    }
   ],
   "source": [
    "print(get_top_n_preds(5, model, \"The historical significance of\", tokenizer))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d086165",
   "metadata": {
    "papermill": {
     "duration": 0.00412,
     "end_time": "2024-03-23T22:58:58.680736",
     "exception": false,
     "start_time": "2024-03-23T22:58:58.676616",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "The untrained heads are predicting somewhat randomly"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a8523cb",
   "metadata": {
    "papermill": {
     "duration": 0.00419,
     "end_time": "2024-03-23T22:58:58.689059",
     "exception": false,
     "start_time": "2024-03-23T22:58:58.684869",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "In the collator, we need to make sure that the labels for each head are padded correctly. Here, we are padding with -100, the ignore_index token for cross_entropy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "06ea9ebc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T22:58:58.706006Z",
     "iopub.status.busy": "2024-03-23T22:58:58.705694Z",
     "iopub.status.idle": "2024-03-23T23:18:37.178058Z",
     "shell.execute_reply": "2024-03-23T23:18:37.177453Z"
    },
    "papermill": {
     "duration": 1178.481053,
     "end_time": "2024-03-23T23:18:37.179369",
     "exception": false,
     "start_time": "2024-03-23T22:58:58.698316",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mykeller\u001b[0m (\u001b[33mchm-hci\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: wandb version 0.16.4 is available!  To upgrade, please run:\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m:  $ pip install wandb --upgrade\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/raven/u/ykeller/transformer_heads/wandb/run-20240323_235900-4fskzy02\u001b[0m\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mvibrant-oath-211\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u2b50\ufe0f View project at \u001b[34m\u001b[4mhttps://wandb.ai/chm-hci/huggingface\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \ud83d\ude80 View run at \u001b[34m\u001b[4mhttps://wandb.ai/chm-hci/huggingface/runs/4fskzy02\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2954' max='2954' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2954/2954 19:30, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>81.295800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>57.871500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>51.464100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>48.580100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>47.124300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>45.641500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>700</td>\n",
       "      <td>44.463100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800</td>\n",
       "      <td>44.077800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>900</td>\n",
       "      <td>43.277400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>42.728300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1100</td>\n",
       "      <td>42.656600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1200</td>\n",
       "      <td>41.909800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1300</td>\n",
       "      <td>41.522000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1400</td>\n",
       "      <td>41.539900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>41.129800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1600</td>\n",
       "      <td>40.734200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1700</td>\n",
       "      <td>40.546000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1800</td>\n",
       "      <td>40.576200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1900</td>\n",
       "      <td>40.721000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>40.208600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2100</td>\n",
       "      <td>39.578400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2200</td>\n",
       "      <td>39.858100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2300</td>\n",
       "      <td>39.662900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2400</td>\n",
       "      <td>39.846800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>39.966600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2600</td>\n",
       "      <td>39.637100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2700</td>\n",
       "      <td>39.699900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2800</td>\n",
       "      <td>39.687400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2900</td>\n",
       "      <td>39.631400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checkpoint destination directory linear_probe_test/checkpoint-500 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checkpoint destination directory linear_probe_test/checkpoint-1000 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checkpoint destination directory linear_probe_test/checkpoint-1500 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checkpoint destination directory linear_probe_test/checkpoint-2000 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Checkpoint destination directory linear_probe_test/checkpoint-2500 already exists and is non-empty.Saving will proceed but saved results may be invalid.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:460: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=2954, training_loss=43.90577940530848, metrics={'train_runtime': 1177.9828, 'train_samples_per_second': 20.057, 'train_steps_per_second': 2.508, 'total_flos': 1.2795692896977408e+16, 'train_loss': 43.90577940530848, 'epoch': 1.0})"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "args = TrainingArguments(\n",
    "    output_dir=\"linear_probe_test\",\n",
    "    learning_rate=0.0002,\n",
    "    num_train_epochs=train_epochs,\n",
    "    logging_steps=logging_steps,\n",
    "    do_eval=False,\n",
    "    remove_unused_columns=False,\n",
    ")\n",
    "collator = DataCollatorWithPadding(\n",
    "    feature_name_to_padding_value={\n",
    "        \"input_ids\": tokenizer.pad_token_id,\n",
    "        \"attention_mask\": 0,\n",
    "        **{key.name: -100 for key in head_configs},\n",
    "    }\n",
    ")\n",
    "trainer = Trainer(\n",
    "    model,\n",
    "    args=args,\n",
    "    train_dataset=dd[\"train\"],\n",
    "    data_collator=collator,\n",
    ")\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ec5354ac",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:18:37.191715Z",
     "iopub.status.busy": "2024-03-23T23:18:37.191228Z",
     "iopub.status.idle": "2024-03-23T23:23:27.277515Z",
     "shell.execute_reply": "2024-03-23T23:23:27.276860Z"
    },
    "papermill": {
     "duration": 290.093636,
     "end_time": "2024-03-23T23:23:27.278629",
     "exception": false,
     "start_time": "2024-03-23T23:18:37.184993",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Evaluating: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 308/308 [04:50<00:00,  1.06it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/ykeller/conda-envs/sh_finetuning/lib/python3.10/site-packages/torch/utils/checkpoint.py:90: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(37.68969941448856, {'wikitext_head_1': 4.537133027206767, 'wikitext_head_3': 4.251083857827372, 'wikitext_head_5': 4.451241332989235, 'wikitext_head_7': 4.719932771348334, 'wikitext_head_9': 4.847185150369421, 'wikitext_head_11': 4.841892783517961, 'wikitext_head_13': 5.563870134291711, 'lm_head': 4.4773603600341})\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "print(evaluate_head_wise(model, dd[\"validation\"], collator, epochs=eval_epochs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2c45396",
   "metadata": {
    "papermill": {
     "duration": 0.018763,
     "end_time": "2024-03-23T23:23:27.354395",
     "exception": false,
     "start_time": "2024-03-23T23:23:27.335632",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "That one is some interesting result. Despite the fact that gpt2 was pretrained to do causal_lm after the last transformer block, the linear probe hooked after the third to last block performs the best for causal_lm on wikitext data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "eed397c4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-03-23T23:23:27.393216Z",
     "iopub.status.busy": "2024-03-23T23:23:27.392752Z",
     "iopub.status.idle": "2024-03-23T23:23:27.423905Z",
     "shell.execute_reply": "2024-03-23T23:23:27.423435Z"
    },
    "papermill": {
     "duration": 0.051993,
     "end_time": "2024-03-23T23:23:27.425016",
     "exception": false,
     "start_time": "2024-03-23T23:23:27.373023",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'wikitext_head_1': [' the', ' his', ' this', ' its', ' a'], 'wikitext_head_3': [' the', ' this', ' his', ' these', ' a'], 'wikitext_head_5': [' the', ' this', ' a', ' his', ' <'], 'wikitext_head_7': [' the', ' this', ' his', ' its', ' a'], 'wikitext_head_9': [' the', ' this', ' <', ' a', ' his'], 'wikitext_head_11': [' the', ' <', ' a', ' his', ' this'], 'wikitext_head_13': [' the', ' a', ' <', ' his', '@'], 'lm_head': [' the', ' this', ' these', ' a', ' his']}\n"
     ]
    }
   ],
   "source": [
    "print(get_top_n_preds(5, model, \"The historical significance of\", tokenizer))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12fc3359",
   "metadata": {
    "papermill": {
     "duration": 0.019003,
     "end_time": "2024-03-23T23:23:27.463133",
     "exception": false,
     "start_time": "2024-03-23T23:23:27.444130",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "The heads are now predicting pretty likely tokens. Note that predicting *<* is an artefact of the wikitext data."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "sh_finetuning",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1507.027661,
   "end_time": "2024-03-23T23:23:30.200121",
   "environment_variables": {},
   "exception": null,
   "input_path": "notebooks/base/multi_linear_probe.ipynb",
   "output_path": "notebooks/gpt2/multi_linear_probe.ipynb",
   "parameters": {
    "eval_epochs": 1,
    "logging_steps": 100,
    "model_path": "gpt2",
    "train_epochs": 1
   },
   "start_time": "2024-03-23T22:58:23.172460",
   "version": "2.5.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "144b0bb972134c589e9c207126aa5af1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "15b3bce62ac54e59afcf63cbeb39ccc8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_ee411f1186514831a6449ff4d88a56ef",
        "IPY_MODEL_acc04770a46d476286df91cc33f9d32e",
        "IPY_MODEL_e92cc796d2c4404e893939d1e0ab1db8"
       ],
       "layout": "IPY_MODEL_ee2f20d6aa8f4a4fa029e1e23a264bdd",
       "tabbable": null,
       "tooltip": null
      }
     },
     "1aa2efc61fea4ea9910d8075ce5a9f4f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_37e6712d26894afbb76c131e3079a2a8",
        "IPY_MODEL_7a1f1bdf84b04c2783c8bef9b7c017ad",
        "IPY_MODEL_6d09cb450b4d4849ad1a397428d9ce2b"
       ],
       "layout": "IPY_MODEL_64870ac971b2454ebcbffe681c9f245b",
       "tabbable": null,
       "tooltip": null
      }
     },
     "294c911575d3449ab879166c113aafad": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "2e157e016353422e830f23c7316d505c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "318e20db69cd449fa222a8fd729c9e43": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "335709902c1c46e89ac8ef914f726e20": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "36078cdc26964285a0133d39c385a30c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "37e6712d26894afbb76c131e3079a2a8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_83630a546c3746adb577a8e77254d537",
       "placeholder": "\u200b",
       "style": "IPY_MODEL_b3b7df5fd64144c2bf0154f4594570c3",
       "tabbable": null,
       "tooltip": null,
       "value": "Map:\u2007100%"
      }
     },
     "493649e57aad4fcb953c5490c26bb67c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5479fd88a3f44227838cf2edba84bb92": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_cec681ce29754675ae9eec52a4c51ba5",
       "max": 2460,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_cda57fb943654c49b7e895ebd0b3b4cc",
       "tabbable": null,
       "tooltip": null,
       "value": 2460
      }
     },
     "5906286f3da2446e881465125b563c28": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_493649e57aad4fcb953c5490c26bb67c",
       "placeholder": "\u200b",
       "style": "IPY_MODEL_ba9131d5e28e492cb643c95e8f3f7968",
       "tabbable": null,
       "tooltip": null,
       "value": "Map:\u2007100%"
      }
     },
     "5978753b72f84558a5341abb872e061a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_d0f3f564aa89445fadbc1f7e7d86a5e8",
       "placeholder": "\u200b",
       "style": "IPY_MODEL_36078cdc26964285a0133d39c385a30c",
       "tabbable": null,
       "tooltip": null,
       "value": "\u20072460/2460\u2007[00:00&lt;00:00,\u20076495.60\u2007examples/s]"
      }
     },
     "603323b806d5480d8244a06631b97459": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "63924785a03e43b9b43eab4e68a8b689": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "64870ac971b2454ebcbffe681c9f245b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6d09cb450b4d4849ad1a397428d9ce2b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_63924785a03e43b9b43eab4e68a8b689",
       "placeholder": "\u200b",
       "style": "IPY_MODEL_144b0bb972134c589e9c207126aa5af1",
       "tabbable": null,
       "tooltip": null,
       "value": "\u200723627/23627\u2007[00:03&lt;00:00,\u20076537.77\u2007examples/s]"
      }
     },
     "7946243478f548079a39ae3b99a14c2b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7a1f1bdf84b04c2783c8bef9b7c017ad": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_7946243478f548079a39ae3b99a14c2b",
       "max": 23627,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_335709902c1c46e89ac8ef914f726e20",
       "tabbable": null,
       "tooltip": null,
       "value": 23627
      }
     },
     "7ea9d676c89c4c949e3e1cc624a73876": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "83630a546c3746adb577a8e77254d537": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9b975852c5574cc1b819e8b07a9c467e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_5906286f3da2446e881465125b563c28",
        "IPY_MODEL_5479fd88a3f44227838cf2edba84bb92",
        "IPY_MODEL_5978753b72f84558a5341abb872e061a"
       ],
       "layout": "IPY_MODEL_f6f5a6bab70c46df81377c5b9c136bb9",
       "tabbable": null,
       "tooltip": null
      }
     },
     "acc04770a46d476286df91cc33f9d32e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_603323b806d5480d8244a06631b97459",
       "max": 2870,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_294c911575d3449ab879166c113aafad",
       "tabbable": null,
       "tooltip": null,
       "value": 2870
      }
     },
     "b3b7df5fd64144c2bf0154f4594570c3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "ba9131d5e28e492cb643c95e8f3f7968": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "background": null,
       "description_width": "",
       "font_size": null,
       "text_color": null
      }
     },
     "cda57fb943654c49b7e895ebd0b3b4cc": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "cec681ce29754675ae9eec52a4c51ba5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d0f3f564aa89445fadbc1f7e7d86a5e8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e8a294182fe84f8dafd4a5837d77932e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e92cc796d2c4404e893939d1e0ab1db8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_e8a294182fe84f8dafd4a5837d77932e",
       "placeholder": "\u200b",
       "style": "IPY_MODEL_2e157e016353422e830f23c7316d505c",
       "tabbable": null,
       "tooltip": null,
       "value": "\u20072870/2870\u2007[00:00&lt;00:00,\u20076219.31\u2007examples/s]"
      }
     },
     "ee2f20d6aa8f4a4fa029e1e23a264bdd": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ee411f1186514831a6449ff4d88a56ef": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "2.0.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "2.0.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "2.0.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_allow_html": false,
       "layout": "IPY_MODEL_318e20db69cd449fa222a8fd729c9e43",
       "placeholder": "\u200b",
       "style": "IPY_MODEL_7ea9d676c89c4c949e3e1cc624a73876",
       "tabbable": null,
       "tooltip": null,
       "value": "Map:\u2007100%"
      }
     },
     "f6f5a6bab70c46df81377c5b9c136bb9": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "2.0.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "2.0.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "2.0.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border_bottom": null,
       "border_left": null,
       "border_right": null,
       "border_top": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}