name_tuning_interpretability.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.lines as mlines\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_transform.bias', 'vocab_projector.bias', 'vocab_projector.weight', 'vocab_transform.weight', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias']\n",
      "- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "######USING ATTENTION STYLE:  frozen-in-time\n"
     ]
    }
   ],
   "source": [
    "vlm = \"miles\"\n",
    "\n",
    "if vlm == \"clip\":\n",
    "    from CLIP.CLIPVLM import ClipVLM as VLM\n",
    "    vlm = VLM(num_frames=10)\n",
    "elif vlm == \"miles\":\n",
    "    from MILES.wrapper import MILES_SimilarityVLM as VLM\n",
    "    vlm = VLM()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset.moma.momaapi.lookup._read_anns() took 8.403647184371948 sec\n",
      "dataset.moma.momaapi.statistics._read_statistics() took 0.022177696228027344 sec\n",
      "dataset.moma.momaapi.lookup._read_anns() took 1.121511697769165 sec\n",
      "dataset.moma.momaapi.statistics._read_statistics() took 0.0010061264038085938 sec\n",
      "dataset.moma.momaapi.lookup._read_anns() took 1.1289994716644287 sec\n",
      "dataset.moma.momaapi.statistics._read_statistics() took 0.0009944438934326172 sec\n",
      "dataset.moma.momaapi.lookup._read_anns() took 1.115957260131836 sec\n",
      "dataset.moma.momaapi.statistics._read_statistics() took 0.0 sec\n",
      "dataset.moma.momaapi.lookup._read_anns() took 1.1237092018127441 sec\n",
      "dataset.moma.momaapi.statistics._read_statistics() took 0.0 sec\n"
     ]
    }
   ],
   "source": [
    "from dataset import DatasetHandler\n",
    "dataset = \"moma_sact\"\n",
    "train_data = DatasetHandler(dataset, split=\"train\")\n",
    "val_data = DatasetHandler(dataset, split=\"val\")\n",
    "test_data = DatasetHandler(dataset, split=\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "one person is on their knees while proposing\n",
      "one person is putting a ring on their partner\n",
      "players are trying to catch a frisbee\n",
      "the adult is chasing after the child on the bike\n",
      "the adult is comforting the child\n",
      "the adult is explaining to the child how to ride a bike\n",
      "the adult is feeding the child\n",
      "the adult is holding the child on the bike\n",
      "the adult is playing with the child\n",
      "the adult is pushing the stroller\n",
      "the adult is wrapping the child or changing the child's diaper or clothes\n",
      "the audience and host are cheering the award recipient on stage\n",
      "the award winner is delivering a speech on stage\n",
      "the barber is applying shampoo\n",
      "the barber is combing hair\n",
      "the barber is cutting hair with a razor\n",
      "the barber is cutting hair with scissors\n",
      "the barber is drying hair\n",
      "the barber is massaging the customer's face or applying products\n",
      "the barber is shaving a beard with a razor\n",
      "the barber is shaving a beard with scissors\n",
      "the barber is washing the customer's hair\n",
      "the basketball player is dribbling or shooting a basket\n",
      "the basketball player is shooting a free throw\n",
      "the basketball players are in a timeout\n",
      "the caregiver is helping the patient out of the bed or chair\n",
      "the caregiver is helping the patient walk down the stairs\n",
      "the child falls off the bike\n",
      "the customers are raising their glasses\n",
      "the driver is being asked to look in a specific direction\n",
      "the driver is receiving food and drink\n",
      "the driver is speaking with the police officer\n",
      "the firefighters are doing other work\n",
      "the firefighters are exiting the firetruck towards the scene\n",
      "the firefighters are extinguishing fire\n",
      "the firefighters are talking\n",
      "the goalkeeper is saving a shot\n",
      "the guests are walking to the front desk\n",
      "the health worker is injecting the patient with the syringe\n",
      "the health worker is stopping the patient's bleeding\n",
      "the health worker is using an alcohol swab on the patient's skin before the injection\n",
      "the host is giving the award winner an award\n",
      "the host is introducing the award winner\n",
      "the massage therapist is applying nail polish to the customer's nails\n",
      "the massage therapist is cleaning the customer's nails\n",
      "the massage therapist is filing the customer's nails\n",
      "the massage therapist is massaging or applying oils or creams to the customer's hands or feet\n",
      "the passengers are walking through the security gate\n",
      "the patient is walking with physical therapy parallel bars\n",
      "the person is being asked to blow into an alcohol tester\n",
      "the person is being asked to walk straight by the police\n",
      "the player is throwing a frisbee\n",
      "the players are passing the basketball\n",
      "the players are passing the soccer ball\n",
      "the receptionist is exchanging items with the guests\n",
      "the receptionist is talking to guests\n",
      "the repairmen are fixing the car's tires\n",
      "the repairmen are leaving while the car is departing\n",
      "the repairmen are watching the car driving towards the repair station\n",
      "the security officer is inspecting a bag\n",
      "the security officer is inspecting a passenger\n",
      "the soccer player is dribbling towards the goal\n",
      "the soccer players are celebrating\n",
      "the student is playing the piano while the teacher watches\n",
      "the table tennis player is hitting the ball into the net which ends the point\n",
      "the table tennis player is serving the ball\n",
      "the table tennis players are not playing the game right now\n",
      "the table tennis players are rallying with each other\n",
      "the teacher and student are playing the piano together\n",
      "the teacher is playing the piano\n",
      "the teacher is verbally instructing the student\n",
      "the tennis table player is failing to return the ball\n",
      "the waiter is cleaning the table\n",
      "the waiter is is talking to the customer or helping them into their seat\n",
      "the waiter is placing utensils on the table\n",
      "the waiters are placing food on the table\n",
      "the waiters are pouring or presenting drinks for the customers\n",
      "two people are hugging\n",
      "two people are kissing\n",
      "two people are talking\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n\".join(train_data.data_dict.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter classes\n",
    "CLASS_SETS = {\n",
    "    \"moma_sact\": [\n",
    "        [\n",
    "            \"the barber is combing hair\",\n",
    "            \"the barber is cutting hair with a razor\",\n",
    "            \"the barber is cutting hair with scissors\",\n",
    "            \"the barber is drying hair\",\n",
    "            \"the barber is washing the customer's hair\"\n",
    "        ],\n",
    "        [\n",
    "            \"two people are hugging\",\n",
    "            \"two people are kissing\",\n",
    "            \"two people are talking\"\n",
    "        ],\n",
    "        [\n",
    "            \"the security officer is inspecting a bag\",\n",
    "            \"the security officer is inspecting a passenger\"\n",
    "        ],\n",
    "        [\n",
    "            \"the firefighters are doing other work\",\n",
    "            \"the firefighters are exiting the firetruck towards the scene\",\n",
    "            \"the firefighters are extinguishing fire\",\n",
    "            \"the firefighters are talking\"\n",
    "        ],\n",
    "        list(val_data.data_dict.keys())\n",
    "    ],\n",
    "    \"smsm\": [\n",
    "        [\n",
    "            \"putting something into something\",\n",
    "            \"dropping something into something\"\n",
    "        ]\n",
    "    ],\n",
    "    \"kinetics_100\": [\n",
    "        [\n",
    "            \"dancing ballet\",\n",
    "            \"dancing charleston\",\n",
    "            \"dancing macarena\"\n",
    "        ]\n",
    "    ]\n",
    "}\n",
    "ALLOWED_CLASSES = CLASS_SETS[dataset][-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_way = len(ALLOWED_CLASSES)\n",
    "n_support = 5\n",
    "\n",
    "class_names = np.array(ALLOWED_CLASSES)\n",
    "support_videos = np.vstack([\n",
    "    np.random.choice(train_data.data_dict[name], size=n_support, replace=False)\n",
    "    for name in ALLOWED_CLASSES\n",
    "])\n",
    "query_videos = np.array([\n",
    "    np.random.choice(test_data.data_dict[name])\n",
    "    for name in ALLOWED_CLASSES\n",
    "])\n",
    "\n",
    "use_val_tuning = False\n",
    "if use_val_tuning:\n",
    "    val_videos = np.concatenate([\n",
    "        val_data.data_dict[name]\n",
    "        for name in ALLOWED_CLASSES\n",
    "    ])\n",
    "    val_labels = np.concatenate([\n",
    "        [i] * len(val_data.data_dict[name])\n",
    "        for i, name in enumerate(ALLOWED_CLASSES)\n",
    "    ])\n",
    "else:\n",
    "    val_videos = None\n",
    "    val_labels = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifiers = {}\n",
    "\n",
    "from classifier.name_tuning import NameTuningFewShotClassifier\n",
    "classifiers[\"clip\"] = NameTuningFewShotClassifier(vlm) # Any of these classifiers can do zero-shot classification\n",
    "classifiers[\"na\"] = NameTuningFewShotClassifier(\n",
    "    vlm,\n",
    "    #prompt_ensemble_id=\"simple_photo\",\n",
    "    lr=5e-3,\n",
    "    name_regularization=0.01,\n",
    "    epochs=20,\n",
    "    batch_size=8\n",
    ")\n",
    "from classifier.coop import CoopFewShotClassifier\n",
    "classifiers[\"coop\"] = CoopFewShotClassifier(\n",
    "    vlm,\n",
    "    lr=5e-3,\n",
    "    epochs=20,\n",
    "    batch_size=8\n",
    ")\n",
    "if False:\n",
    "    from classifier.cona import CoNaFewShotClassifier\n",
    "    classifiers[\"cona\"] = CoNaFewShotClassifier(\n",
    "        vlm,\n",
    "        lr=1e-3,\n",
    "        name_regularization=1,\n",
    "        epochs=20,\n",
    "        batch_size=8\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------ clip ------\n",
      "------ na ------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\ProgramData\\Anaconda3\\envs\\vlm_env\\lib\\site-packages\\torch\\optim\\lr_scheduler.py:163: UserWarning: The epoch parameter in `scheduler.step()` was not necessary and is being deprecated where possible. Please use `scheduler.step()` to step the scheduler. During the deprecation, if epoch is different from None, the closed form is used instead of the new chainable form, where available. Please open an issue if you are unable to replicate your use case: https://github.com/pytorch/pytorch/issues/new/choose.\n",
      "  warnings.warn(EPOCH_DEPRECATION_WARNING, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch     0: Support Acc = 0.315, Loss = 2.380E+00, Reg Loss = 1.641E-09, Combined Loss = 2.380E+00\n",
      "Epoch     1: Support Acc = 0.332, Loss = 2.174E+00, Reg Loss = 2.935E-04, Combined Loss = 2.174E+00\n",
      "Epoch     2: Support Acc = 0.387, Loss = 1.826E+00, Reg Loss = 1.166E-03, Combined Loss = 1.828E+00\n",
      "Epoch     3: Support Acc = 0.405, Loss = 1.733E+00, Reg Loss = 2.008E-03, Combined Loss = 1.735E+00\n",
      "Epoch     4: Support Acc = 0.442, Loss = 1.583E+00, Reg Loss = 2.940E-03, Combined Loss = 1.586E+00\n",
      "Epoch     5: Support Acc = 0.460, Loss = 1.516E+00, Reg Loss = 3.903E-03, Combined Loss = 1.520E+00\n",
      "Epoch     6: Support Acc = 0.502, Loss = 1.419E+00, Reg Loss = 4.761E-03, Combined Loss = 1.424E+00\n",
      "Epoch     7: Support Acc = 0.492, Loss = 1.403E+00, Reg Loss = 5.581E-03, Combined Loss = 1.409E+00\n",
      "Epoch     8: Support Acc = 0.548, Loss = 1.312E+00, Reg Loss = 6.363E-03, Combined Loss = 1.318E+00\n",
      "Epoch     9: Support Acc = 0.553, Loss = 1.304E+00, Reg Loss = 6.982E-03, Combined Loss = 1.311E+00\n",
      "Epoch    10: Support Acc = 0.548, Loss = 1.309E+00, Reg Loss = 7.580E-03, Combined Loss = 1.317E+00\n",
      "Epoch    11: Support Acc = 0.577, Loss = 1.261E+00, Reg Loss = 8.063E-03, Combined Loss = 1.269E+00\n",
      "Epoch    12: Support Acc = 0.567, Loss = 1.227E+00, Reg Loss = 8.452E-03, Combined Loss = 1.236E+00\n",
      "Epoch    13: Support Acc = 0.575, Loss = 1.219E+00, Reg Loss = 8.750E-03, Combined Loss = 1.228E+00\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[287], line 6\u001b[0m\n\u001b[0;32m      4\u001b[0m     classifier\u001b[39m.\u001b[39mpredict(class_names, \u001b[39mNone\u001b[39;00m, query_videos, val_videos, val_labels) \u001b[39m# zero-shot classifier (i.e. default CLIP)\u001b[39;00m\n\u001b[0;32m      5\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m----> 6\u001b[0m     classifier\u001b[39m.\u001b[39;49mpredict(class_names, support_videos, query_videos, val_videos, val_labels)\n",
      "File \u001b[1;32md:\\non_onedrive_code\\vlm_benchmark\\classifier\\name_tuning.py:192\u001b[0m, in \u001b[0;36mNameTuningFewShotClassifier.predict\u001b[1;34m(self, category_names, support_video_paths, query_video_paths, val_tuning_video_paths, val_tuning_video_labels)\u001b[0m\n\u001b[0;32m    190\u001b[0m \u001b[39mfor\u001b[39;00m batch_idx, (vid_paths, vid_labels) \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(train_dataloader):\n\u001b[0;32m    191\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrandom_augment:\n\u001b[1;32m--> 192\u001b[0m         vid_embeds \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mcat([\n\u001b[0;32m    193\u001b[0m             torch\u001b[39m.\u001b[39mfrom_numpy(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mvlm\u001b[39m.\u001b[39mvideo_encoder(vid_path, random_augment\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m))\u001b[39m.\u001b[39munsqueeze(\u001b[39m0\u001b[39m)\u001b[39m.\u001b[39mto(DEVICE)\n\u001b[0;32m    194\u001b[0m             \u001b[39mfor\u001b[39;00m vid_path \u001b[39min\u001b[39;00m vid_paths\n\u001b[0;32m    195\u001b[0m         ], dim\u001b[39m=\u001b[39m\u001b[39m0\u001b[39m)\n\u001b[0;32m    196\u001b[0m     \u001b[39melse\u001b[39;00m: \u001b[39m# Use version of video encoder which can cache results for fast lookup\u001b[39;00m\n\u001b[0;32m    197\u001b[0m         vid_embeds \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mcat([\n\u001b[0;32m    198\u001b[0m             torch\u001b[39m.\u001b[39mfrom_numpy(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mvlm\u001b[39m.\u001b[39mget_video_embeds(vid_path))\u001b[39m.\u001b[39munsqueeze(\u001b[39m0\u001b[39m)\u001b[39m.\u001b[39mto(DEVICE)\n\u001b[0;32m    199\u001b[0m             \u001b[39mfor\u001b[39;00m vid_path \u001b[39min\u001b[39;00m vid_paths\n\u001b[0;32m    200\u001b[0m         ], dim\u001b[39m=\u001b[39m\u001b[39m0\u001b[39m)\n",
      "File \u001b[1;32md:\\non_onedrive_code\\vlm_benchmark\\classifier\\name_tuning.py:193\u001b[0m, in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    190\u001b[0m \u001b[39mfor\u001b[39;00m batch_idx, (vid_paths, vid_labels) \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(train_dataloader):\n\u001b[0;32m    191\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrandom_augment:\n\u001b[0;32m    192\u001b[0m         vid_embeds \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mcat([\n\u001b[1;32m--> 193\u001b[0m             torch\u001b[39m.\u001b[39mfrom_numpy(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mvlm\u001b[39m.\u001b[39;49mvideo_encoder(vid_path, random_augment\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m))\u001b[39m.\u001b[39munsqueeze(\u001b[39m0\u001b[39m)\u001b[39m.\u001b[39mto(DEVICE)\n\u001b[0;32m    194\u001b[0m             \u001b[39mfor\u001b[39;00m vid_path \u001b[39min\u001b[39;00m vid_paths\n\u001b[0;32m    195\u001b[0m         ], dim\u001b[39m=\u001b[39m\u001b[39m0\u001b[39m)\n\u001b[0;32m    196\u001b[0m     \u001b[39melse\u001b[39;00m: \u001b[39m# Use version of video encoder which can cache results for fast lookup\u001b[39;00m\n\u001b[0;32m    197\u001b[0m         vid_embeds \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mcat([\n\u001b[0;32m    198\u001b[0m             torch\u001b[39m.\u001b[39mfrom_numpy(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mvlm\u001b[39m.\u001b[39mget_video_embeds(vid_path))\u001b[39m.\u001b[39munsqueeze(\u001b[39m0\u001b[39m)\u001b[39m.\u001b[39mto(DEVICE)\n\u001b[0;32m    199\u001b[0m             \u001b[39mfor\u001b[39;00m vid_path \u001b[39min\u001b[39;00m vid_paths\n\u001b[0;32m    200\u001b[0m         ], dim\u001b[39m=\u001b[39m\u001b[39m0\u001b[39m)\n",
      "File \u001b[1;32md:\\non_onedrive_code\\vlm_benchmark\\MILES\\wrapper.py:161\u001b[0m, in \u001b[0;36mMILES_SimilarityVLM.video_encoder\u001b[1;34m(self, video_path, subvideo_start_frame, subvideo_end_frame, random_augment)\u001b[0m\n\u001b[0;32m    159\u001b[0m video_len \u001b[39m=\u001b[39m \u001b[39mlen\u001b[39m(video_reader)\n\u001b[0;32m    160\u001b[0m frame_indices \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39msample_frame_indices(video_len, subvideo_start_frame, subvideo_end_frame, random_augment)\n\u001b[1;32m--> 161\u001b[0m frames \u001b[39m=\u001b[39m video_reader\u001b[39m.\u001b[39;49mget_batch(frame_indices)\u001b[39m.\u001b[39mnumpy()\n\u001b[0;32m    163\u001b[0m \u001b[39m# Transform\u001b[39;00m\n\u001b[0;32m    164\u001b[0m frames \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mfrom_numpy(frames)\u001b[39m.\u001b[39mfloat() \u001b[39m/\u001b[39m \u001b[39m255\u001b[39m\n",
      "File \u001b[1;32md:\\ProgramData\\Anaconda3\\envs\\vlm_env\\lib\\site-packages\\decord\\video_reader.py:175\u001b[0m, in \u001b[0;36mVideoReader.get_batch\u001b[1;34m(self, indices)\u001b[0m\n\u001b[0;32m    173\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_handle \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m    174\u001b[0m indices \u001b[39m=\u001b[39m _nd\u001b[39m.\u001b[39marray(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_validate_indices(indices))\n\u001b[1;32m--> 175\u001b[0m arr \u001b[39m=\u001b[39m _CAPI_VideoReaderGetBatch(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_handle, indices)\n\u001b[0;32m    176\u001b[0m \u001b[39mreturn\u001b[39;00m bridge_out(arr)\n",
      "File \u001b[1;32md:\\ProgramData\\Anaconda3\\envs\\vlm_env\\lib\\site-packages\\decord\\_ffi\\_ctypes\\function.py:173\u001b[0m, in \u001b[0;36mFunctionBase.__call__\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m    171\u001b[0m ret_val \u001b[39m=\u001b[39m DECORDValue()\n\u001b[0;32m    172\u001b[0m ret_tcode \u001b[39m=\u001b[39m ctypes\u001b[39m.\u001b[39mc_int()\n\u001b[1;32m--> 173\u001b[0m check_call(_LIB\u001b[39m.\u001b[39;49mDECORDFuncCall(\n\u001b[0;32m    174\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mhandle, values, tcodes, ctypes\u001b[39m.\u001b[39;49mc_int(num_args),\n\u001b[0;32m    175\u001b[0m     ctypes\u001b[39m.\u001b[39;49mbyref(ret_val), ctypes\u001b[39m.\u001b[39;49mbyref(ret_tcode)))\n\u001b[0;32m    176\u001b[0m _ \u001b[39m=\u001b[39m temp_args\n\u001b[0;32m    177\u001b[0m _ \u001b[39m=\u001b[39m args\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for name, classifier in classifiers.items():\n",
    "    print(f\"------ {name} ------\")\n",
    "    if name == \"clip\":\n",
    "        classifier.predict(class_names, None, query_videos, val_videos, val_labels) # zero-shot classifier (i.e. default CLIP)\n",
    "    else:\n",
    "        classifier.predict(class_names, support_videos, query_videos, val_videos, val_labels)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classifier accuracy and tuned class tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------ clip Class Probs ------\n",
      "Accuracy: 0.4\n",
      "Average correct-class prob: 0.397\n",
      "[[0.074 0.11  0.078 0.059 0.679]\n",
      " [0.13  0.583 0.233 0.024 0.03 ]\n",
      " [0.117 0.45  0.373 0.022 0.038]\n",
      " [0.168 0.166 0.144 0.144 0.378]\n",
      " [0.049 0.075 0.029 0.032 0.814]]\n",
      "------ na Class Probs ------\n",
      "Accuracy: 0.6\n",
      "Average correct-class prob: 0.430\n",
      "[[0.016 0.011 0.013 0.114 0.846]\n",
      " [0.148 0.552 0.144 0.099 0.057]\n",
      " [0.117 0.247 0.442 0.073 0.121]\n",
      " [0.535 0.04  0.041 0.307 0.076]\n",
      " [0.026 0.045 0.018 0.079 0.832]]\n",
      "------ coop Class Probs ------\n",
      "Accuracy: 0.6\n",
      "Average correct-class prob: 0.423\n",
      "[[0.042 0.055 0.051 0.112 0.739]\n",
      " [0.077 0.605 0.197 0.034 0.086]\n",
      " [0.072 0.327 0.372 0.075 0.155]\n",
      " [0.246 0.102 0.109 0.265 0.278]\n",
      " [0.034 0.053 0.032 0.051 0.831]]\n"
     ]
    }
   ],
   "source": [
    "for name, classifier in classifiers.items():\n",
    "    print(f\"------ {name} Class Probs ------\")\n",
    "    print(\"Accuracy:\", (classifier.query_class_probabilities.argmax(axis=-1) == np.arange(len(class_names))).mean())\n",
    "    print(f\"Average correct-class prob: {classifier.query_class_probabilities[np.arange(len(class_names)), np.arange(len(class_names))].mean():.3f}\")\n",
    "    print(classifier.query_class_probabilities.round(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the  (0.008) barber  (0.024) is  (0.012) comb (0.032) ing  (0.019) hair  (0.013)\n",
      "the  (0.008) barber  (0.026) is  (0.012) cutting  (0.015) hair  (0.013) with  (0.008) a  (0.006) razor  (0.018)\n",
      "the  (0.007) barber  (0.022) is  (0.008) cutting  (0.014) hair  (0.012) with  (0.008) scissors  (0.022)\n",
      "the  (0.012) barber  (0.029) is  (0.011) drying  (0.026) hair  (0.015)\n",
      "the  (0.012) barber  (0.046) is  (0.018) washing  (0.029) the  (0.012) customer  (0.020) 's  (0.009) hair  (0.018)\n",
      "0 (0.471) 0 (0.453) sundaywithmarsha  (0.441) flyeagles (0.438) instaweather (0.463) 0 (0.463) 0 (0.444) 0 (0.462) 0 (0.449) 0 (0.464) 0 (0.452) 0 (0.454) flyeagles (0.456) 0 (0.459) flyeagles (0.441) 0 (0.432) the  (0.000) barber  (0.000) is  (0.000) comb (0.000) ing  (0.000) hair  (0.000)\n",
      "0 (0.471) 0 (0.453) sundaywithmarsha  (0.441) flyeagles (0.438) instaweather (0.463) 0 (0.463) 0 (0.444) 0 (0.462) 0 (0.449) 0 (0.464) 0 (0.452) 0 (0.454) flyeagles (0.456) 0 (0.459) flyeagles (0.441) 0 (0.432) the  (0.000) barber  (0.000) is  (0.000) cutting  (0.000) hair  (0.000) with  (0.000) a  (0.000) razor  (0.000)\n",
      "0 (0.471) 0 (0.453) sundaywithmarsha  (0.441) flyeagles (0.438) instaweather (0.463) 0 (0.463) 0 (0.444) 0 (0.462) 0 (0.449) 0 (0.464) 0 (0.452) 0 (0.454) flyeagles (0.456) 0 (0.459) flyeagles (0.441) 0 (0.432) the  (0.000) barber  (0.000) is  (0.000) cutting  (0.000) hair  (0.000) with  (0.000) scissors  (0.000)\n",
      "0 (0.471) 0 (0.453) sundaywithmarsha  (0.441) flyeagles (0.438) instaweather (0.463) 0 (0.463) 0 (0.444) 0 (0.462) 0 (0.449) 0 (0.464) 0 (0.452) 0 (0.454) flyeagles (0.456) 0 (0.459) flyeagles (0.441) 0 (0.432) the  (0.000) barber  (0.000) is  (0.000) drying  (0.000) hair  (0.000)\n",
      "0 (0.471) 0 (0.453) sundaywithmarsha  (0.441) flyeagles (0.438) instaweather (0.463) 0 (0.463) 0 (0.444) 0 (0.462) 0 (0.449) 0 (0.464) 0 (0.452) 0 (0.454) flyeagles (0.456) 0 (0.459) flyeagles (0.441) 0 (0.432) the  (0.000) barber  (0.000) is  (0.000) washing  (0.000) the  (0.000) customer  (0.000) 's  (0.000) hair  (0.000)\n"
     ]
    }
   ],
   "source": [
    "def most_similar_words(word_embedding: torch.Tensor, k=1):\n",
    "    token_embeds = vlm.model.text_model.embeddings.token_embedding.weight\n",
    "    token_embeds = F.normalize(token_embeds, dim=1)\n",
    "    word_embedding = F.normalize(word_embedding, dim=0)\n",
    "    similarities = token_embeds @ word_embedding\n",
    "    \n",
    "    best = torch.topk(similarities, k, largest=True)\n",
    "    similarities = best.values.cpu().tolist()\n",
    "    tokens = best.indices\n",
    "    words = [vlm.tokenizer.decode(t) for t in tokens]\n",
    "    \n",
    "    return words, similarities\n",
    "\n",
    "def closest_words(word_embedding: torch.Tensor, k=1):\n",
    "    distances = torch.norm(vlm.model.text_model.embeddings.token_embedding.weight - word_embedding.unsqueeze(0), dim=1)\n",
    "    \n",
    "    best = torch.topk(distances, k, largest=False)\n",
    "    distances = best.values.cpu().tolist()\n",
    "    tokens = best.indices\n",
    "    words = [vlm.tokenizer.decode(t) for t in tokens]\n",
    "    \n",
    "    return words, distances\n",
    "\n",
    "def print_tuned_names(classifier, k=1):\n",
    "    tuned_name_embeds, attn_mask = classifier.tuned_input_embeds\n",
    "    for category_ind in range(n_way):\n",
    "        n_tokens = attn_mask[category_ind].sum().type(torch.long)\n",
    "        start_token_ind = vlm.text_start_special_token_count()\n",
    "        end_token_ind = n_tokens - vlm.text_end_special_token_count()\n",
    "        \n",
    "        # Print a line for most similar replacements, then next most similar, etc\n",
    "        lines = [[] for _ in range(k)]\n",
    "        for token_ind in range(start_token_ind, end_token_ind):\n",
    "            # words, values = most_similar_words(tuned_name_embeds[category_ind, token_ind], k)\n",
    "            words, values = closest_words(tuned_name_embeds[category_ind, token_ind], k)\n",
    "            for i in range(k):\n",
    "                lines[i] += [f\"{words[i]} ({values[i]:.3f})\"]\n",
    "                #lines[i] += [words[i]]\n",
    "        for line in lines:\n",
    "            print(\" \".join(line))\n",
    "\n",
    "for name, classifier in classifiers.items():\n",
    "    if name == \"clip\":\n",
    "        continue\n",
    "    print_tuned_names(classifier)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Output Text Embedding Comparison Diagram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_vids_in_sne = True\n",
    "perplexity = 30\n",
    "\n",
    "\n",
    "# Record saved as {class name -> [orig text embed, tuned text embed epoch 0, epoch 1, ...]}\n",
    "orig_text_embeds_per_category = np.array([vlm.get_text_embeds(name) for name in class_names])\n",
    "na_text_embeds_per_category = np.array([classifiers[\"na\"].text_embed_training_record[name][-1] for name in class_names])\n",
    "coop_text_embeds_per_category = np.array([classifiers[\"coop\"].text_embed_training_record[name][-1] for name in class_names])\n",
    "\n",
    "support_video_embeds = np.array([\n",
    "    [\n",
    "        vlm.get_video_embeds(path)\n",
    "        for path in paths\n",
    "    ]\n",
    "    for paths in support_videos\n",
    "])\n",
    "\n",
    "# TSNE transform\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.manifold import TSNE\n",
    "sklearn_metric = \"cosine\"\n",
    "\n",
    "stacked_embeddings = []\n",
    "stacked_embeddings.append(orig_text_embeds_per_category)\n",
    "stacked_embeddings.append(na_text_embeds_per_category)\n",
    "stacked_embeddings.append(coop_text_embeds_per_category)\n",
    "if use_vids_in_sne:\n",
    "    stacked_embeddings.append(support_video_embeds.reshape(n_way * n_support, -1))\n",
    "stacked_embeddings = np.concatenate(stacked_embeddings, axis=0)\n",
    "sne_embeddings = TSNE(n_components=2, metric=sklearn_metric, perplexity=perplexity).fit_transform(stacked_embeddings)\n",
    "\n",
    "next_ind = 0\n",
    "orig_text_sne = sne_embeddings[next_ind : next_ind + n_way]\n",
    "next_ind += n_way\n",
    "na_text_sne = sne_embeddings[next_ind : next_ind + n_way]\n",
    "next_ind += n_way\n",
    "coop_text_sne = sne_embeddings[next_ind : next_ind + n_way]\n",
    "next_ind += n_way\n",
    "if use_vids_in_sne:\n",
    "    support_video_sne = sne_embeddings[next_ind : next_ind + n_way * n_support].reshape(n_way, n_support, 2)\n",
    "    next_ind += n_way * n_support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAekAAAFhCAYAAABULVIiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABFEUlEQVR4nO3deXwTZf4H8M8kPdObtKWFnhxtKVeByiFyg0WOpazIVZFLVEQBQeWnyLULHiuX7iosN7gFBIXKssqNyiKngC60QIGWlpXSUuh9J8/vj5hsQw9C25Bp83m/XnlBZp6Z+SZN8+kz88yMJIQQICIiItlRWLoAIiIiqhxDmoiISKYY0kRERDLFkCYiIpIphjQREZFMMaSJiIhkiiFNREQkUwxpIiIimWJIExERyRRDuhK//vorJk6ciODgYDg4OMDZ2RkdO3bEX/7yF9y7d8/Qrnfv3ujdu7flCq2CJEmQJAkffvhhhXmbNm2CJEk4e/asBSqrmeTkZMNretgjOTm5TrYZHx+PhQsXmrw+/fta1eP777+vk7oA3eeuTZs2dba+6gQFBWHChAkPbad//eXfL7n+fjzo7t27sLe3r/b34vDhw4iMjISTkxMkSUJcXBy2bt2KlStXPpYav//+e5N/B2qrNj+3hQsX1kkN9D82li5AbtauXYtXX30VoaGheOuttxAeHo7S0lKcPXsWq1evxokTJ7B7925Ll2mSDz/8EC+99BIaNWpk6VJqxdfXFydOnDCa9uqrryI7OxuxsbEV2taF+Ph4LFq0CL1790ZQUJDJy23cuBFhYWEVpoeHh9dJXfXJ559/bukSTPLFF1+gpKQEALB+/XpERkYazRdCYOTIkQgJCcGePXvg5OSE0NBQjBs3DhcvXsTMmTPNXmPHjh0r/A4MHz4czZs3x9KlS+t0W7X5ub344osYOHBgHVZDDOlyTpw4galTp2LAgAGIi4uDvb29Yd6AAQMwe/Zs7Nu3z4IVmq5///74/vvvsWTJEixbtszS5dSKvb09unbtajTN1dUVJSUlFaZbWps2bSp8yVur+vKHyYYNG+Dt7Y3AwEBs27YNy5cvh6Ojo2H+b7/9hnv37mH48OHo16+f2espLCw02j6g+7w/+Fm3t7eHu7t7tb8DQggUFRVVWF91avNz8/Pzg5+fX42Xp4q4u7uc999/H5IkYc2aNUYBrWdnZ4c//OEP1a5j0aJF6NKlCxo1agRXV1d07NgR69evx4P3MTly5Ah69+4NtVoNR0dHBAQE4Nlnn0VBQYGhzapVq9C+fXs4OzvDxcUFYWFhePfdd016LaGhoZg8eTI+++wz3Lx5s9q2Z8+exejRoxEUFARHR0cEBQVhzJgxFZbT79I8cuQIpkyZArVaDVdXV7zwwgvIz89HWloaRo4cCXd3d/j6+uLNN99EaWmp0TpKSkqwePFihIWFwd7eHl5eXpg4cSIyMjJMel3VycnJwZtvvong4GDY2dmhadOmmDlzJvLz8w1tXnnlFTg4OODnn382TNNqtejXrx8aN26M27dvY9OmTXjuuecAAH369DHsRty0aVOtawR0hyNee+01bNy4EaGhoXB0dERkZCROnjwJIQQ+/vhjBAcHw9nZGX379sW1a9cqXc+xY8fQtWtXODo6omnTppg3bx40Go1RG1Pf79LSUrz99tvw8fGBSqXCU089hdOnT1e63ZMnT6J79+5wcHBAkyZN8M4771T4OQMVd5vqD1ssXboUy5cvN7zGbt264eTJkxWWX7t2LUJCQmBvb4/w8HBs3boVEyZMqLBnoza/J6dOncLFixcxbtw4TJkyBdnZ2fj6668N8xcuXGgInTlz5kCSJAQFBaF3797417/+hZs3b1a6q9nU9z0oKAhDhgzBrl270KFDBzg4OGDRokUm1V4Z/Wdr9erVaNWqFezt7bF582YApn831ebnVtnubv1r3LdvHzp27AhHR0eEhYVhw4YNFer/97//jW7dusHBwcHwmV63bl2dHsqqdwQJIYQoKysTKpVKdOnSxeRlevXqJXr16mU0bcKECWL9+vXi4MGD4uDBg+LPf/6zcHR0FIsWLTK0SUpKEg4ODmLAgAEiLi5OfP/99yI2NlaMGzdO3L9/XwghxLZt2wQA8frrr4sDBw6IQ4cOidWrV4vp06c/tC4AYtq0aeL27dtCpVKJcePGGeZt3LhRABBnzpwxTNu5c6eYP3++2L17t/jhhx/E9u3bRa9evYSXl5fIyMiosGxwcLCYPXu2OHDggPjoo4+EUqkUY8aMER07dhSLFy8WBw8eFHPmzBEAxLJlywzLazQaMXDgQOHk5CQWLVokDh48KNatWyeaNm0qwsPDRUFBwSO9961btzY8z8/PFxEREcLT01MsX75cHDp0SHzyySfCzc1N9O3bV2i1WiGEEIWFhSIiIkI0a9bM8F7Pnz9fKBQKceDAASGEEOnp6eL9998XAMRnn30mTpw4IU6cOCHS09OrrEf/3pw8eVKUlpYaPcrKyir8fAIDA8WTTz4pdu3aJXbv3i1CQkJEo0aNxBtvvCGGDRsm9u7dK2JjY0Xjxo1Fu3btDPXrX7tarRZNmjQRn376qdi/f7+YPn264edek/d7/PjxQpIk8dZbb4kDBw6I5cuXi6ZNmwpXV1cxfvx4Q7tLly4JlUolwsPDxbZt28Q333wjoqKiREBAgAAgkpKSjOos//uRlJQkAIigoCAxcOBAERcXJ+Li4kTbtm2Fh4eHyMrKMrT9+9//LgCIZ5991vBehISEiMDAQBEYGGhoV5vfEyGEmDJligAgLl26JHJycoRKpRK9e/c2zE9NTRW7du0ybOPEiRPi3Llz4tKlS6J79+7Cx8fH8Pk4ceLEI7/vgYGBwtfXVzRr1kxs2LBBHD16VJw+fdqk2gMDA8XgwYONpgEQTZs2Fe3atRNbt24VR44cERcvXhRCmPbdJETtfm4LFiwQD8ZKYGCg8PPzE+Hh4WLLli1i//794rnnnhMAxA8//GBo98svvwgHBwfRrl07sX37drFnzx4xaNAgERQUVOGzZU0Y0r9LS0sTAMTo0aNNXqaykC5Po9GI0tJS8ac//Umo1WrDF+1XX30lAIgLFy5Uuexrr70m3N3dTa6lvPJf1nPnzhUKhUL88ssvQojKQ/pBZWVlIi8vTzg5OYlPPvnEMF2/7Ouvv27UPjo6WgAQy5cvN5oeEREhOnbsaHiu/0L9+uuvjdqdOXNGABCff/65ya/xwZD+4IMPhEKhqPC69O/1t99+a5iWmJgoXF1dRXR0tDh06JBQKBTivffeM1pu586dAoA4evSoSfXo35vKHkql0qgtAOHj4yPy8vIM0+Li4gQAERERYRTIK1euFADEr7/+avTaAYhvvvnGaL1TpkwRCoVC3Lx5Uwhh+vudkJAgAIg33njDqF1sbKwAYBTSo0aNEo6OjiItLc0wraysTISFhZkc0m3btjX6w+X06dMCgNi2bZsQQvd74+PjU+EP5ps3bwpbW1ujkK7N70l+fr5wdXUVXbt2NUzT/7Fy7dq1CnV//PHHRssPHjzYqBa9R/mcBwYGCqVSKa5cufLI9VcV0m5ubuLevXvVLlvVd5MQNf+5CVF1SDs4OBg+l0Lo/lhu1KiRePnllw3TnnvuOeHk5GTUMdBoNCI8PNyqQ5q7u+vYkSNH0L9/f7i5uUGpVMLW1hbz589HZmYm0tPTAQARERGws7PDSy+9hM2bN+PGjRsV1tO5c2dkZWVhzJgx+Oabb3D37t0a1fP222+jUaNGmDNnTpVt8vLyMGfOHLRo0QI2NjawsbGBs7Mz8vPzkZCQUKH9kCFDjJ63atUKADB48OAK08vvMt+7dy/c3d0xdOhQlJWVGR4RERHw8fGp1QjovXv3ok2bNoiIiDBad1RUVIXR1S1atMDatWsRFxeHIUOGoEePHli4cGGNt13eli1bcObMGaPHqVOnKrTr06cPnJycDM/17+EzzzxjtLtQP/3BQw8uLi4VDr2MHTsWWq0WP/74IwDT3++jR48CAGJiYozWN3LkSNjYGA9bOXr0qOHQgJ5SqcSoUaMe/ub8bvDgwVAqlYbn7dq1M3qNV65cMRw6KS8gIADdu3c3mlab35MdO3YgJycHkyZNMkybNGkShBDYuHGjyet50KN+ztu1a4eQkJAab+9Bffv2hYeHR4Xppnw3VedhP7fqREREICAgwPDcwcEBISEhRsv+8MMP6Nu3Lzw9PQ3TFApFhc+BtWFI/87T0xMqlQpJSUk1Xsfp06fx9NNPA9AdTzt+/DjOnDmDuXPnAtANCAGA5s2b49ChQ/D29sa0adPQvHlzNG/eHJ988olhXePGjcOGDRtw8+ZNPPvss/D29kaXLl1w8ODBR6rJ1dUV7733Hvbt22f4Mn7Q2LFj8be//Q0vvvgi9u/fj9OnT+PMmTPw8vIy1Fzeg6PF7ezsqpxeVFRkeH7nzh1kZWXBzs4Otra2Ro+0tLQa/yGiX/evv/5aYb0uLi4QQlRY9+DBg9G4cWMUFRVh1qxZRl8+tdGqVStERkYaPTp16lSh3aO8hwCM3kcARiGp5+PjAwDIzMwEYPr7rW+vX17PxsYGarXaaFpmZmaFdpUtW50H16kf/6H/rOnrqew1PjitNr8n69evh4ODAwYOHIisrCxkZWWhXbt2CAoKwqZNmyoc3zfVo37O6+qMhOrWZ+p3U3Ue9nN7lGX1y5dfNjMz06SfubXh6O7fKZVK9OvXD9999x1u3bpVoxGK27dvh62tLfbu3QsHBwfD9Li4uApte/TogR49ekCj0eDs2bP461//ipkzZ6Jx48YYPXo0AGDixImYOHEi8vPz8eOPP2LBggUYMmQIrl69isDAQJPrmjp1Kj755BPMmTMHU6dONZqXnZ2NvXv3YsGCBfi///s/w/Ti4mKjc8LrgqenJ9RqdZUj5F1cXGq1bkdHx0oHo+jnl/fKK68gNzcXrVu3xvTp09GjR49Kex9ydefOnQrT0tLSAPzvC9HU91vfPi0tDU2bNjXMLysrMwSmnlqtNmynsm3XBX091b3G8mrye3L16lX8+9//BgCjHl55+/fvx6BBgx65/kf9nNf1ecWVre9RvpssRa1Wm/wztyYM6XLeeecdfPvtt5gyZQq++eYbQy9Gr7S0FPv27cPQoUMrXV6SJNjY2Bj1ygoLC/HFF19UuU2lUokuXbogLCwMsbGxOHfunCGk9ZycnPDMM8+gpKQE0dHRuHTp0iOFtJ2dHRYvXoyYmJgKYSVJEoQQFUazr1u3rsY9iaoMGTIE27dvh0ajQZcuXep83e+//z7UajWCg4Orbbtu3Tr84x//wIYNG9CrVy907NgREydONPrCepRegiXk5uZiz549Rru8t27dCoVCgZ49ewIw/f3Wj+SNjY016vXv2LEDZWVlRm379OmDPXv24M6dO4YejkajwZdffllXLw2hoaHw8fHBjh07MGvWLMP0lJQU/PTTT2jSpEmlyz3K78n69esB6HqVLVq0MJpXWFiIYcOGYcOGDdWG9IM9QT1zfs5rqibfTY9br1698O233+Lu3buG7ymtVoudO3dauDLLYkiX061bN6xatQqvvvoqOnXqhKlTp6J169YoLS3F+fPnsWbNGrRp06bKkB48eDCWL1+OsWPH4qWXXkJmZiaWLl1aIQBXr16NI0eOYPDgwQgICEBRUZGhB9i/f38AwJQpU+Do6Iju3bvD19cXaWlp+OCDD+Dm5oYnnnjikV/bmDFjsHTpUnz33XdG011dXdGzZ098/PHH8PT0RFBQEH744QesX78e7u7uj7yd6owePRqxsbEYNGgQZsyYgc6dO8PW1ha3bt3C0aNHMWzYMAwfPrxG6545cya+/vpr9OzZE2+88QbatWsHrVaLlJQUHDhwALNnz0aXLl3wn//8B9OnT8f48eMxceJEALov7BEjRmDlypWGC1Por+i1Zs0auLi4wMHBAcHBwZXutivv4sWLFYIN0B3i8PLyqtFrq4xarcbUqVORkpKCkJAQfPvtt1i7di2mTp1q6Bma+n63atUKzz//PFauXAlbW1v0798fFy9exNKlS+Hq6mq03ffeew979uxB3759MX/+fKhUKnz22WdGp7nVlkKhwKJFi/Dyyy9jxIgRmDRpErKysrBo0SL4+vpCofjfUbqa/J6UlZVhy5YtaNWqFV588cVK2wwdOhR79uyp9tTAtm3bYteuXVi1ahU6deoEhUKByMhIs37Oa8rU7yZLmjt3Lv75z3+iX79+mDt3LhwdHbF69WrDZ6v8z92qWHjgmixduHBBjB8/XgQEBAg7Ozvh5OQkOnToIObPn290Gk5lo7s3bNggQkNDhb29vWjWrJn44IMPxPr1641GJ544cUIMHz5cBAYGCnt7e6FWq0WvXr3Enj17DOvZvHmz6NOnj2jcuLGws7MTTZo0ESNHjjQa5VsVPHAqjt6BAwcMI47Lj4K+deuWePbZZ4WHh4dwcXERAwcOFBcvXhSBgYFGI3urGhmuH9FZflSmELqRsk5OTkbTSktLxdKlS0X79u2Fg4ODcHZ2FmFhYeLll18WiYmJD31teg+O7hZCiLy8PPHee++J0NBQYWdnJ9zc3ETbtm3FG2+8IdLS0kReXp4ICwsT4eHhIj8/32jZadOmCVtbW3Hq1CnDtJUrV4rg4GChVCoFALFx48Yq66ludDcAsXbtWkPbyn4+VY0gPnr0qAAgdu7cWeG1f//99yIyMlLY29sLX19f8e6774rS0lKj5U19v4uLi8Xs2bOFt7e3cHBwEF27dhUnTpyo8BkQQojjx4+Lrl27Cnt7e+Hj4yPeeustsWbNGpNHdz/4GvXvyYIFC4ymrVmzRrRo0ULY2dmJkJAQsWHDBjFs2DDRoUMHQ5ua/J7oR9KvXLmyyjb79u0znEJYVd337t0TI0aMEO7u7kKSJKNRzaa+75WN0DZVVaO7K/vdF8K07yYhavdzq2p0d2WvsbLvz2PHjokuXboYfbY++ugjAcDoVC9rIgnxwJnsREQylJWVhZCQEERHR2PNmjWWLocek6effhrJycm4evWqpUuxCO7uJiLZSUtLw5IlS9CnTx+o1WrcvHkTK1asQG5uLmbMmGHp8shMZs2ahQ4dOsDf3x/37t1DbGwsDh48aBhDYI0Y0kQkO/b29khOTsarr76Ke/fuQaVSoWvXrli9ejVat25t6fLITDQaDebPn4+0tDRIkoTw8HB88cUXeP755y1dmsVwdzcREZFMWelwOSIiIvljSBMREckUQ5qIiEimajxwTKvV4rfffoOLi0udX9aOiIioIRNCIDc3F02aNKn2Qi01DunffvsN/v7+NV2ciIjI6qWmplZ7r4gah7T+IvGpqakVLh1IREREVcvJyYG/v/9DbyxU45DW7+J2dXVlSBMREdXAww4Xc+AYERGRTDGkiYiIZIohTUREJFMMaSIiIpliSBMREckUQ5qIiEimGNJEREQyxZAmIiKSqRpfzISoISu+kYSy9HTYeHvDvlmwpcshIivFkCZ6QNbu3bgfuxXaggIoVCp4xIyF+/Dhli6LiKwQd3cTlVN8Iwn3Y7cCQsDWzw8QAvdjt6L4RpKlSyMiK8SQJiqnLD0d2oICKNVqSAoFlGo1tAUFKEtPt3RpRGSFGNJE5dh4e0OhUkGTmQmh1UKTmQmFSgUbb29Ll0ZEVoghTVSOfbNgeMSMBSQJpbduAZIEj5ixHDxGRBbBgWNED3AfPhyO7SM4upuILI4hTVQJ+2bBDGcisjju7iYiIpIphjQREZFMMaSJiIhkiiFNREQkUwxpIiIimWJIExERyRRDmoiISKYY0kRERDLFkCYiIpIphjQREZFMMaSJiIhkiiFNREQkUwxpIiIimWJIExERyRRDmoiISKYY0kRERDLFkCYiIpIphjQREZFMMaSJiIhkiiFNREQkUwxpIiIimWJIExERyRRDmoiISKYY0kRERDLFkCYiIpIphjQREZFM2Vi6ACJTFN9IQll6Omy8vWHfLNjS5RARPRYMaZK9rN27cT92K7QFBVCoVPCIGQv34cMtXRYRkdlxdzfJWvGNJNyP3QoIAVs/P0AI3I/diuIbSZYujYjI7BjSJGtl6enQFhRAqVZDUiigVKuhLShAWXq6pUsjIjI7hjTJmo23NxQqFTSZmRBaLTSZmVCoVLDx9rZ0aaa7mwgk/aj7l4joETCkSdbsmwXDI2YsIEkovXULkCR4xIytP4PHzscCu14C/jVb9+/5WEtXRET1CAeOkey5Dx8Ox/YR9W90991E4Mw6QGgB9yAgP1333L8z4NnS0tURUT3AkKZ6wb5ZcP0JZ73c20BJni6gFQrAyRvIStZNZ0gTkQm4u5vIXFx8ATtnXQ9aq9X9a+esm05EZAKGNJG5eLYEnngRkBS6HrSk0D1nL5qITMTd3UTm1CFGdww697auB82AJqJHwJAmMjfPlgxnIqoR7u4mIiKSKYY0ERGRTDGkiYiIZIohTUREJFMMaSIiIpliSBMREckUQ5qIiEimGNJEREQyxZAmIiKSKYY0ERGRTDGkiYiIZIohTUREJFMMaSIiIpliSBMREckUQ5qIiEimGNJEREQyxZAmIiKSKYY0ERGRTDGkiYiIZIohTUREJFMMaSIiIpliSBMREckUQ5qIiEimGNJEREQyxZAmIiKSKYY0ERGRTDGkiYiIZIohTUREJFMMaSIiACgpAE6t0f1LJBMMaSIiADi1GvjuLeD03y1dCZEBQ5qIqDgXOL5S9/9/rwCK8yxaDpEeQ5qI6PRaoChH9/+iHODMWsvWQ/Q7hjQRWTdDL1r8PkGwN02ywZAmIutWvhetx940yYSNpQsgInosMq/res3llRQAx5bhf71ovd97009MgVarRNbXu+D+7B+hcHR8XNUSAWBIE5E1yLwO/LXjoy3ze2/6XrwzMpYvh7awAJ5TppinPqIqcHc3ETV8D/agTSKgOboCmWvXAAAy16yFNj+/busiegiGNBFRFe7/RwNtnm4AmTYvD/e2brVwRWRtGNIkL3cTgaQfdf/KeZ3U4GlKJWQmOJcb9C3Ym6bHjsekST7OxwJn1gEleYCdM/DEi0CHGPmtkxo+hQ3uX1NBWyoZTdb3pnlsmh4X9qRJHu4m6sJUaAH3IN2/Z9bVrvdrjnWSVdA0G4TMK+4AjEOavWl63BjSJA+5t3W9XSdvQKHQ/VuSp5sup3WSVbif1gLaYk2l83hsmh4nhjTJg4uvbnd0fjqg1er+tXPWTZfTOqnB05RKyNy2BxAPnjv9O/am6TFiSJM8eLbUHS+WFEBWsu7fJ17UTZfTOqleKr6ZYnLb+4lO0BZUf7tK9qbpceHAMZKPDjGAf2fd7mgX37oJU3Osk+qdglMnYW9Cuwojuqvye2+60dixUDg51UWJRJViSJO8eLas+yA1xzqpXtHkl5rULjvZEdpSBSBJgFJZdUMhoM3NRdY336DR2LF1VCVRRQxpImrwhFsQrn3hA6WirNp22jIJCjst7MLaw7Ftu+pXKklQdepUh1USVcSQJqIGz2XAANz9299QCjuT2vt2TIfDlDGAurmZKyOqHgeOEVGD5xAaAscOHarfhQ0AkoCjZzEcin8F1vbT3ZiDyIIY0kRkFZp8+AEUzs5VB7UkoLAVaNIlCxAaoDgHiJv6WGskehBDmoisgl1gIIJ3fAnHVi10EyTxvwcAR3UJggdkwM7l94uYCA2Qegq4c8lCFRPxmDQRWRG7wEAEvdYVRd/8hNxUW2hLFFDYaeHiVwQH90oGlUlKIGEv0Lj14y+WCAxpIrI2Rdlw8NDCwS3v4W0lBVCUZfaSiKrC3d1EZF0c3HQ3WzGF0AIO7mYth6g6DGkisi6thuqON5tCaHTtiSyEIU1E1qVxa8Cvs+54c3UkJeDfBWgc/njqIqoEQ5qIrM/w1YC9a9VBLSl186NXPd66iB7AkCYi66NuDkw5DPhF6p5LSkBh+7/Q9ovUzecVx8jCOLqbiKyTujkw+YDuPOiEvbpR3A7uumPQ3MVNMsGQJiLr1rg1z4Mm2eLubiIiIpliSBMREckUQ5qIiEimGNJEREQyxYFj1HDcTQRybwMuvoBnS0tXQ0RUawxpahjOxwJn1gEleYCdM/DEi0CHGEtXRURUK9zdTfXf3URdQAst4B6k+/fMOt10IqJ6jCFN9V/ubV0P2skbUCh0/5bk6aYTEdVj3N1N9Z+Lr24Xd366LqDz03XPXXwtXRnVMyXJydDk5z+0ndLJCXZBQeYviKweQ5rqP8+WumPQZ9YBWcn/OybNwWP0CEqSk3F94DMmt2++7zsGNZkdQ5oahg4xgH9nju6mGjOlB12b9kQ1wZCmhsOzJcOZiBoUhjTR74pvJKEsPR023t6wbxZs6XKIiBjSRACQtXs37sduhbagAAqVCh4xY+E+fLilyyIiK8dTsMjqFd9Iwv3YrYAQsPXzA4TA/ditKL6RZOnSiMjKMaTJ6pWlp0NbUAClWg1JoYBSrYa2oABl6emWLo2IrBxDmqyejbc3FCoVNJmZEFotNJmZUKhUsPH2tnRpRGTlGNJk9eybBcMjZiwgSSi9dQuQJHjEjOXgMSKyOA4cIwLgPnw4HNtHcHQ3EckKQ5rod/bNghnOVkzp5GTW9kQ1wZAmIgJgFxSE5vu+47W7SVYY0kREv2Pwktxw4BgREZFMMaSJiIhkiiFNREQkUwxpIiIimWJIExERyRRDmoiISKZ4ChZZVFJ2EjIKMuCl8kKwGy8kQkRUHkOaLCbuWhy2X96OgtICqGxVGB02GtEtoi1dFhGRbHB3N1lEUnYStl/eDgEBPxc/CAhsv7wdSdm8hzMRkR5DmiwioyADBaUFUDuooZAUUDuoUVBagIyCDEuXRkQkGwxpsggvlRdUtipkFmVCK7TILMqEylYFL5WXpUsjIpINhjRZRLBbMEaHjYYECbdyb0GChNFhozl4jIioHA4cI4uJbhGN9l7tObqbiKgKDGmyqGC3YJPCufhGEsrS02Hj7c17PhOR1WBIk+xl7d6N+7FboS0ogEKlgkfMWLgPH27psoiIzI7HpEnWim8k4X7sVkAI2Pr5AULgfuxWFN/gqVpE1PAxpEnWytLToS0ogFKthqRQQKlWQ1tQgLL0dEuXRkRkdgxpkjUbb28oVCpoMjMhtFpoMjOhUKlg4+1t6dKIiMyOIU2yZt8sGB4xYwFJQumtW4AkwSNmLAePEZFV4MAxkj334cPh2D6Co7uJyOowpKlesG8WzHAmIqvD3d1EREQyxZAmIiKSKYY0ERGRTPGYNBF42VEikieGNFk9XnaUiOSKu7vJqvGyo0QkZwxpsmq87CgRyRlDmqwaLztKRHLGkCarxsuOEpGcceAYWT1edpSI5IohTQRedpSI5Im7u4mIiGSKIU1ERCRTDGkiIiKZYkgTERHJFEOaiIhIpji6m6zX3UQg9zbg4gt4trR0NUREFTCkyTqdjwXOrANK8gA7Z+CJF4EOMZauiojICHd3U4OWlJ2E07dPIym73A0z7ibqAlpoAfcg3b9n1ummExHJCHvS1GDFXYvD9svbUVBaAJWtCqPDRiO6RbRuF3dJni6gFQrAyRvIStZN525vIpIR9qSpQUrKTsL2y9shIODn4gcBge2Xt+t61C6+ul3c+emAVqv7185ZN52ISEbYk6YGKaMgAwWlBfBz8YNCUkDtoMat3FvIKMhAsG9n3THoM+t0PWj9MWn2oq3S5bQc7LuYhpzCMrg62mBgGx+E+bhauiwiAAxpaqC8VF5Q2aqQWZQJtYMamUWZUNmq4KXy0jXoEAP4d+bobiuWfDcfs3ZcwLmULCgVEhQSoBXAykOJ6BTogWXPtUeQp5OlyyQrx93d1CAFuwVjdNhoSJBwK/cWJEgYHTYawW7lbqLh2RII7smAtkLJd/Mx7LPj+OVWNgBAoxUo1QhotAIAcCE1C8M+O47ku/mWLJOIPWlquKJbRKO9V3tkFGTAS+VlHNBk1WbtuIC84jJDKD9IoxXIKy7Dmzt/wVdTn3zM1RH9D0OaGrRgt2CGMxm5nJaDcylZD22n0QqcvXkfl9NyeIyaLIa7u4nIquy7mAalQjKprVIhYf/FO2auiKhqDGkisio5hWUwMaOhkIDswlLzFkRUDYY0EVkVV0cbVHEougKtANwcbc1bEFE1GNJEZFUGtvGpcsDYgzRagYFtfMxcEVHVGNJEZFXCfFzRMcD9ocellQoJkYEeCPVxeUyVEVXEkCYiq7N8ZASc7W2qDGqlQoKzvQ2WPtf+MVdGZIwhTURWJ8jTCd9M644O/u4AdKFsq5QMod3B3x3fTOvOK46RxfE8aSKySkGeTvhq6pO4nJaD/RfvILuwFG6OthjYxoe7uEk2GNJkFknZSbzSF9ULYT6uvFgJyRZDmupclfdxJiKiR8Jj0lSnqr2PMxERPRKGNNUp/X2c1Q5qw32cC0oLkFGQYenSiB6LwrJCbE3YisKyQkuXQg0AQ5rqVPn7OGuFtuJ9nIkauNiEWHxw+gNsTdhq6VKoAWBIU50y6T7ORA1Ufmk+NvxnAwBg/X/Wo6C0wMIVUX3HgWNU5x7nfZyvZ+ThTk4RGrs6oLmXs9m2Q2SKbZe3Ia80DwCQV5qHbZe3YXLbyRauiuozSQhh4qXmjeXk5MDNzQ3Z2dlwdeXpC/T4fXU2FVtO3kR+cRmc7G3wQtdAjIj0t3RZZKXyS/MxYOcA5JbmGqa52Lrg0HOHoLJVWbAykiNTM5S7u6leup6Rhy0nb0IIgQAPFYQQ2HLyJq5n5Fm6NLJS5XvRevreNFFNMaSpXrqTU4T84jJ4OtlDoZDg6WSP/OIy3MkpsnRpZIX0x6IFjHdMCggem6ZaYUhTvdTY1QFO9ja4m18MrVbgbn4xnOxt0NjVwdKlkRWqrBetx9401QZDmuql5l7OeKFrICRJQsr9AkiShBe6BnLwGD12VfWi9dibptrg6G6qt0ZE+qNDoAdHd5NFVdeL1uNIb6op9qSpXmvu5Ywnm3syoMkiHtaL1mNvmmqKIU0E3Wjxn67f5ehweiR7ru9BbmkulJISNgobKCVllW1zS3Ox6dKmx1ccNQjc3U1Wj+dbU0119O6IsWFjISCQW5KL/cn7oRXaSnvWEiRsubQFQ5oNQYBrgAWqpfqIPWmyajzfmmojtFEo3unyDt7t8i5u5d6qMqAB3S7vIk0R3jv+3mOukuozhjRZNZ5vTXXh6v2ruJBxARqhqbadRmhwPv08rt6/+pgqo/rO7Lu7NRoNSktLzb0ZslJ2dnZQKHR/ayZlJz3y9cLLn2/t6WTP862pRg7fPAylpHxoSAOAUlLicMphhHiEPIbKqL4zW0gLIZCWloasrCxzbYIICoUCwcHB+DblW2y/vB0FpQVQ2aowOmw0oltEP3R5/fnWW07eRMr9AsMxaY4Wp0eRU5IDSZLwkEHeAABJkpBTnGP+oqhBMFtI6wPa29sbKpVK9wEmqkNarRa//fYbrt68iu1XtkNAwM/FD5lFmdh+eTvae7U3qUfN862ptlztXGHqvYqEEHC1502JyDRmCWmNRmMIaLVabY5NEAEAvLy8cCPlBqAB1E5qKCQF1A5q3Mq9hYyCDJN3ezf3cmY4U431C+yHz3/53KS2GqFB/4D+Zq6IGgqzDBzTH4NWqXh7NjIvOzs72Chs4GHngcyiTGiFFplFmVDZquCl8rJ0eWQlQjxCEOEVUe150oDueHQH7w5o6dHyMVVG9Z1ZB45xFzeZmyRJsFHYYHCzwfhH4j9wK/eW4Zh0Zb3o6xl5Ve7WrsnAM2oYjt9MwL2C3Ie2a6RyQffAVpXOW/LUEoz51xjkl+ZXOoBMKSnhZOuExd0X17pesh68mAk1CP0D+6O1T+tqQ7a6i5bEXYur0cAzqv+O30zAK9+PNLn96t47Kg3qANcAbBu8De8dfw/n089DKSkhSRKEENAIDdp5tcPi7ot5IRN6JAzpOpScnIzg4GCcP38eERERJi2zadMmzJw5s05HwdekjoYg2C24yh7wgxctuZtfjC0nb6JDoAcUdhnYfrnmA8+ofjOlB21q+wDXAGx5Zguu3r+KwymHkVOcA1d7V/QP6M9d3FQjsg5pjUaDY8eO4fbt2/D19UWPHj2gVFZ/zKcupKamYuHChfjuu+9w9+5d+Pr6Ijo6GvPnz692IJy/vz9u374NT09Pk7c1atQoDBo0qC7KpmroL1oS4KEyXLQk5X4B7uQUwUaVgYLSAvi5+NV44BlReSEeITwPmuqEbK84tmvXLgQFBaFPnz4YO3Ys+vTpg6CgIOzatcus271x4wYiIyNx9epVbNu2DdeuXcPq1atx+PBhdOvWDffu3at0uZKSEiiVSvj4+MDGxvS/fRwdHeHt7V1X5VMVyl+0RKsVRhct8VJ5QWWr4sAzIpIdWYb0rl27MGLECNy6dcto+n//+1+MGDHCrEE9bdo02NnZ4cCBA+jVqxcCAgLwzDPP4NChQ/jvf/+LuXPnAgCCgoKwePFiTJgwAW5ubpgyZQqSk5MhSRIuXLhgWN+ePXvQsmVLODo6ok+fPti8eTMkSTLs3t60aRPc3d0N7RcuXIiIiAh88cUXCAoKgpubG0aPHo3c3P/tYtu3bx+eeuopuLu7Q61WY8iQIbh+/brZ3pOGQH/REkmSkHK/AJIkGS5aEuwWjNFhoyFBwq3cW5AgVTnwjIjocZLd7m6NRoMZM2ZUemEAIQQkScLMmTMxbNiwOt/1fe/ePezfvx9LliyBo6Oj0TwfHx/ExMTgyy+/xOef686H/PjjjzFv3jy8917lF8xPTk7GiBEjMGPGDLz44os4f/483nzzzYfWcf36dcTFxWHv3r24f/8+Ro4ciQ8//BBLliwBAOTn52PWrFlo27Yt8vPzMX/+fAwfPhwXLlwwXCKTKqruoiXRLaLR3qs9R3cTkazILqSPHTtWoQddnhACqampOHbsGHr37l2n205MTIQQAq1aVX6KRatWrXD//n1kZGQAAPr27WsUusnJyUbtV69ejdDQUHz88ccAgNDQUFy8eNEQtlXRarXYtGkTXFxcAADjxo3D4cOHDcs9++yzRu3Xr18Pb29vxMfHo02bNqa/YCtU3UVLqht4RkRkCbLrdt2+fbtO29Ulfe9ef/53ZGRkte2vXLmCJ554wmha586dH7qdoKAgQ0ADgK+vL9LT0w3Pr1+/jrFjx6JZs2ZwdXVFcLAuWFJSUkx7IUREVC/ILqR9fX3rtN2jaNGiBSRJQnx8fKXzL1++DA8PD8PobScnp2rXp989/+C0h7G1tTV6LkkStFqt4fnQoUORmZmJtWvX4tSpUzh16hQA3eA1IiJqOGQX0j169ICfn1+VVyuTJAn+/v7o0aNHnW9brVZjwIAB+Pzzz1FYWGg0Ly0tDbGxsRg1apTJV1ILCwvDmTNnjKadPXu2VjVmZmYiISEB7733Hvr162fYBU9ERA2P7EJaqVTik08+AVDxsqL65ytXrjTb+dJ/+9vfUFxcjKioKPz4449ITU3Fvn37MGDAADRt2vShx5PLe/nll3H58mXMmTMHV69exY4dO7Bp0yYANb9kqoeHB9RqNdasWYNr167hyJEjmDVrVo3WRUS6S32asz1Rbchu4BgA/PGPf8RXX32FGTNmGA0i8/Pzw8qVK/HHP/7RbNtu2bIlzp49i4ULF2LUqFHIzMyEj48PoqOjsWDBAjRq1MjkdQUHB+Orr77C7Nmz8cknn6Bbt26YO3cupk6dCnt7+xrVp1AosH37dkyfPh1t2rRBaGgoPv300zofREdkLboHtsLq3jtqfe1uInOQhKk3QX1ATk4O3NzckJ2dDVdX43ujFhUVISkpCcHBwXBwcKhxcZa64pg5LVmyBKtXr0ZqaqqlS2kQ6uqzRkT0OFWXoeXJsietp1Qq630P8fPPP8cTTzwBtVqN48eP4+OPP8Zrr71m6bKIiKgekHVINwSJiYlYvHgx7t27h4CAAMyePRvvvPOOpcsiIqJ6gCFtZitWrMCKFSssXQYREdVDshvdTURERDoMaSKiR1BYosHmn5JRWKKxdClkBRjSRESPYONPSViw5xI2/ZRs6VLICjCkiYhMlFdchtXf624Lu+r7a8gvLrNwRdTQMaSJiEy05UQycot0wZxbVIYtJ25auCJq6BjSREQm0Pei9Vd/EmBvmsyPIU1EZILyvWg99qbJ3GQZ0ikpKTh37lyVD3PfNzktLQ2vv/46mjVrBnt7e/j7+2Po0KE4fPiwyesoLCzEggULEBoaCnt7e3h6emLEiBG4dOmSGSsnInN4sBetx940mZvsLmaSkpKC0NBQFBUVVdnGwcEBV65cQUBAQJ1vPzk5Gd27d4e7uzv+8pe/oF27digtLcX+/fsxbdo0XL58+aHrKC4uRv/+/ZGSkoJly5ahS5cuuHPnDj744AN06dIFhw4dQteuXeu8diIyj8p60Xr63vTU3s0fc1VkDWTXk7579261AQ3obqpw9+5ds2z/1VdfhSRJOH36NEaMGIGQkBC0bt0as2bNwsmTJwHo/pAYNmwYnJ2d4erqipEjR+LOnTuGdaxcuRInTpzA3r17MXLkSAQGBqJz5874+uuv0apVK0yePBn6+5pMmDAB0dHRWLRoEby9veHq6oqXX34ZJSUlZnl9RPRoqupF67E3TeYku5C2pHv37mHfvn2YNm0anJycKsx3d3eHEALR0dG4d+8efvjhBxw8eBDXr1/HqFGjDO22bt2KAQMGoH379kbLKxQKvPHGG4iPj8cvv/ximH748GEkJCTg6NGj2LZtG3bv3o1FixaZ74USkcmq60Xr8dg0mQtDupxr165BCIGwsLAq2xw6dAi//vortm7dik6dOqFLly744osv8MMPP+DMmTMAgKtXr6JVq8rvOauffvXqVcM0Ozs7bNiwAa1bt8bgwYPxpz/9CZ9++im0Wm0dvjoielQP60XrsTdN5sKQLke/C1qSpCrbJCQkwN/fH/7+/oZp4eHhcHd3R0JCQo220b59e6hUKsPzbt26IS8vj/ecJrKwXeduIaeoDEqFBJtqHkqFhJyiMuw6d8vSJVMDI7uBY5bUsmVLSJKEhIQEREdHV9pGCFFpiJefHhISgvj4+EqX1w88a9my5UPrqe6PBSIyvyeCGmHCk0GGP66rI0kSIoMaPYaqyJowpMtp1KgRoqKi8Nlnn2H69OkVjktnZWUhPDwcKSkpSE1NNfSm4+PjkZ2dbdiVPXr0aMydOxe//PKL0XFprVaLFStWIDw83Gj6L7/8gsLCQjg6OgIATp48CWdnZ/j5+Zn7JRNRNVr5umLhH1pbugyyYtzd/YDPP/8cGo3GMBo7MTERCQkJ+PTTT9GtWzf0798f7dq1Q0xMDM6dO4fTp0/jhRdeQK9evRAZGQkAeOONN9C5c2cMHToUO3fuREpKCs6cOYNnn30WCQkJWL9+vVEvuaSkBJMnT0Z8fDy+++47LFiwAK+99hoUCv54iIismexSwNPTEw4ODtW2cXBwgKenp1m2HxwcjHPnzqFPnz6YPXs22rRpgwEDBuDw4cNYtWoVJElCXFwcPDw80LNnT/Tv3x/NmjXDl19+aVTfkSNHMH78eLz77rto0aIFBg4cCKVSiZMnT1Y4R7pfv35o2bIlevbsiZEjR2Lo0KFYuHChWV4fERHVH5Iw5WBLJXJycuDm5obs7Gy4uroazSsqKkJSUhKCg4MfGriVSUlJqfY8aE9PT7NcyMQSJkyYgKysLMTFxVm6lHqptp81IiJLqC5Dy5PlMemAgIAGE8JEREQ1Jbvd3URERKQjy560Ndm0aZOlSyAiIpliT5qIiEimGNJEREQyxZAmIiKSKYY0ERGRTDGkiYiIZIohTUREJFMMaarWpk2b4O7ubukyiIisEkO6nAkTJkCSJHz44YdG0+Pi4mR520h9vdU9amvUqFG4evVqHVRLRESPSvYhfejQIYSHh+PQoUOPZXsODg746KOPcP/+/ceyvdr45JNPcPv2bcMDADZu3FhhWm04OjrC29u71ushIqJHJ+uQFkLg3XffRUJCAt59912TbrxeW/3794ePjw8++OCDKttkZmZizJgx8PPzg0qlQtu2bbFt2zajNr1798brr7+OmTNnwsPDA40bN8aaNWuQn5+PiRMnwsXFBc2bN8d3331ntFx8fDwGDRoEZ2dnNG7cGOPGjavyZiNubm7w8fExPADA3d3d8NzX17fCjTvc3d0NVzlLTk6GJEnYtWsX+vTpA5VKhfbt2+PEiROG9g/u7l64cCEiIiLwxRdfICgoCG5ubhg9ejRyc3MNbXJzcxETEwMnJyf4+vpixYoV6N27N2bOnFnle0pERBXJOqQPHDiAM2fOAADOnDmDAwcOmH2bSqUS77//Pv7617/i1q1blbYpKipCp06dsHfvXly8eBEvvfQSxo0bh1OnThm127x5Mzw9PXH69Gm8/vrrmDp1Kp577jk8+eSTOHfuHKKiojBu3DgUFBQAAG7fvo1evXohIiICZ8+exb59+3Dnzh2MHDnSrK957ty5ePPNN3HhwgWEhIRgzJgxKCsrq7L99evXERcXh71792Lv3r344YcfjA4RzJo1C8ePH8eePXtw8OBBHDt2DOfOnTPrayAiapBEDWVnZwsAIjs7u8K8wsJCER8fLwoLC2u6eqHVasUTTzwhlEqlACCUSqV44oknhFarrfE6H2b8+PFi2LBhQgghunbtKiZNmiSEEGL37t3iYW/VoEGDxOzZsw3Pe/XqJZ566inD87KyMuHk5CTGjRtnmHb79m0BQJw4cUIIIcS8efPE008/bbTe1NRUAUBcuXLlofUDELt3767yuRBCuLm5iY0bNwohhEhKShIAxLp16wzzL126JACIhIQEIYQQGzduFG5ubob5CxYsECqVSuTk5BimvfXWW6JLly5CCCFycnKEra2t2Llzp2F+VlaWUKlUYsaMGQ99DY+qLj5rRESPW3UZWp5se9L6XrRGowEAaDSax9abBoCPPvoImzdvRnx8fIV5Go0GS5YsQbt27aBWq+Hs7IwDBw4gJSXFqF27du0M/1cqlVCr1Wjbtq1hWuPGjQEA6enpAICff/4ZR48ehbOzs+ERFhYGQNd7NZfydfr6+hrVVJmgoCC4uLgYLaNvf+PGDZSWlqJz586G+W5ubggNDa3rsomIGjxZ3gVLCIF58+ZBqVQaQhrQBd28efPw9NNPm320dc+ePREVFYV3330XEyZMMJq3bNkyrFixAitXrkTbtm3h5OSEmTNnoqSkxKidra2t0XNJkoym6V+DVqs1/Dt06FB89NFHFerRh+ejkCSpwnH80tLSCu2qq6kylb0ufXv99h78+TxYBxERPZwsQ7r8sejyyvemo6KizF7Hhx9+iIiICISEhBhNP3bsGIYNG4bnn38egC7QEhMT0apVq1ptr2PHjvj6668RFBQEG5va/2i8vLyMRngnJiYajn+bS/PmzWFra4vTp0/D398fAJCTk4PExET06tXLrNsmImpoZLe7W9+LVigqL02hUGDevHmPpWfWtm1bxMTE4K9//avR9BYtWuDgwYP46aefkJCQgJdffhlpaWm13t60adNw7949jBkzBqdPn8aNGzdw4MABTJo0yWiPgqn69u2Lv/3tbzh37hzOnj2LV155pUIvuK65uLhg/PjxeOutt3D06FFcunQJkyZNgkKhkOW55kREcia7kC4pKUFKSkqVu1u1Wi1SU1Mr7Fo2lz//+c8V/iCYN28eOnbsiKioKPTu3Rs+Pj6Ijo6u9baaNGmC48ePQ6PRICoqCm3atMGMGTPg5uZW5R8t1Vm2bBn8/f3Rs2dPjB07Fm+++SZUKlWt63yY5cuXo1u3bhgyZAj69++P7t27o1WrVnBwcDD7tomIGhJJ1LBLmpOTAzc3N2RnZ8PV1dVoXlFREZKSkhAcHFyjL+bU1FRkZGRUOd/b2xt+fn6PvF6yjPz8fDRt2hTLli3D5MmT63Tdtf2sERFZQnUZWp4sj0n7+/sbjmdS/XP+/HlcvnwZnTt3RnZ2Nv70pz8BAIYNG2bhyoiI6hdZhjTVf0uXLsWVK1dgZ2eHTp064dixY/D09LR0WURE9QpDmupchw4d8PPPP1u6DCKiek92A8eIiIhIhyFNREQkUwxpIiIimWJIExERyRRDmoiISKY4upuolq5n5OFOThEauzqguZezpcshogaEPWkz6927N2bOnGly++TkZEiShAsXLpitpkchSRLi4uKqnC+3eh+3r86m4o0vL2Be3EW88eUFfHU21dIlEVEDwpD+nSRJ1T4evF2lqXbt2oU///nPJrf39/fH7du30aZNmxptzxQ///wzJEnCv//970rnR0VF4Q9/+AMA4Pbt23jmmWfMVkt9dj0jD1tO3oQQAgEeKgghsOXkTVzPyLN0aUTUQHB39+/K39Lxyy+/xPz583HlyhXDNEdHR6P2paWlJt1RqlGjRo9Uh1KphI+PzyMt86g6deqE9u3bY+PGjXjqqaeM5qWmpuLQoUPYtWsXAJi9lvrsTk4R8ovLEOChgkIhwdPJHin3C3Anp4i7vYmoTsi+J33r1i0cPXoUt27dMut2fHx8DA83NzdIkmR4XlRUBHd3d+zYsQO9e/eGg4MD/vGPfyAzMxNjxoyBn58fVCoV2rZti23bthmt98Hd3UFBQXj//fcxadIkuLi4ICAgAGvWrDHMf3D38ffffw9JknD48GFERkZCpVLhySefNPoDAgAWL14Mb29vuLi44MUXX8T//d//ISIiosrXO3nyZOzYsQP5+flG0zdt2gQvLy8MHjwYQMXd3adPn0aHDh3g4OCAyMhInD9/vsK64+PjMWjQIDg7O6Nx48YYN24c7t69a5hfXFyM6dOnw9vbGw4ODnjqqaeM7h9+//59xMTEwMvLC46OjmjZsiU2btxY5WuxlMauDnCyt8Hd/GJotQJ384vhZG+Dxq680QcR1Q1Zh/T69esRGBiIvn37IjAwEOvXr7doPXPmzMH06dORkJCAqKgoFBUVoVOnTti7dy8uXryIl156CePGjcOpU6eqXc+yZcsMAffqq69i6tSpuHz5crXLzJ07F8uWLcPZs2dhY2ODSZMmGebFxsZiyZIl+Oijj/Dzzz8jICAAq1atqnZ9MTExKC0txc6dOw3ThBDYtGkTxo8fDxubijtZ8vPzMWTIEISGhuLnn3/GwoUL8eabbxq1uX37Nnr16oWIiAicPXsW+/btw507dzBy5EhDm7fffhtff/01Nm/ejHPnzqFFixaIiorCvXv3AOhuBRofH4/vvvsOCQkJWLVqlSyv+93cyxkvdA2EJElIuV8ASZLwQtdA9qKJqO6IGsrOzhYARHZ2doV5hYWFIj4+XhQWFtZ09SI1NVUoFAoBwPBQKpUiNTW1xus01caNG4Wbm5vheVJSkgAgVq5c+dBlBw0aJGbPnm143qtXLzFjxgzD88DAQPH8888bnmu1WuHt7S1WrVpltK3z588LIYQ4evSoACAOHTpkWOZf//qXAGB4f7t06SKmTZtmVEf37t1F+/btq6111KhRomfPnobnR44cEQDE5cuXDdMAiN27dwshhPj73/8uGjVqJPLz8w3zV61aZVTvvHnzxNNPP220ndTUVAFAXLlyReTl5QlbW1sRGxtrmF9SUiKaNGki/vKXvwghhBg6dKiYOHFitbXr1cVnrbaupeeK49cyxLX0XIvVQET1S3UZWp5se9KJiYnQarVG0zQaDa5du2ahioDIyEij5xqNBkuWLEG7du2gVqvh7OyMAwcOICUlpdr1tGvXzvB//W719PR0k5fx9fUFAMMyV65cQefOnY3aP/i8MpMnT8aPP/5oeE83bNiA7t27IzQ0tNL2CQkJaN++PVQqlWFat27djNr8/PPPOHr0KJydnQ2PsLAwAMD169dx/fp1lJaWonv37oZlbG1t0blzZyQkJAAApk6diu3btyMiIgJvv/02fvrpp4e+Fktq7uWMJ5t7sgdNRHVOtiHdsmVLKBTG5SmVSrRo0cJCFQFOTk5Gz5ctW4YVK1bg7bffxpEjR3DhwgVERUWhpKSk2vU8OOBMkqQKf5BUt4wkSQBgtIx+mp4Qotr1AUD//v0RGBiITZs2IScnB7t27cLkyZOrbG/KOrVaLYYOHYoLFy4YPRITE9GzZ0/DOiqrVz/tmWeewc2bNzFz5kz89ttv6NevX4Xd6kRE1kC2Ie3n54c1a9ZAqVQC0AX03//+d/j5+Vm4sv85duwYhg0bhueffx7t27dHs2bNkJiY+NjrCA0NxenTp42mnT179qHLSZKEiRMnYvPmzdi6dSsUCoXRseMHhYeH45dffkFhYaFh2smTJ43adOzYEZcuXUJQUBBatGhh9HByckKLFi1gZ2dndPpXaWkpzp49i1atWhmmeXl5YcKECfjHP/6BlStXGg2uIyKyFrINaUC3OzY5ORlHjx5FcnJytb08S2jRogUOHjyIn376CQkJCXj55ZeRlpb22Ot4/fXXsX79emzevBmJiYlYvHgxfv311wq91cpMnDgRv/32G959912MHj26wt6C8saOHQuFQoHJkycjPj4e3377LZYuXWrUZtq0abh37x7GjBmD06dP48aNGzhw4AAmTZoEjUYDJycnTJ06FW+99Rb27duH+Ph4TJkyBQUFBYaf7/z58/HNN9/g2rVruHTpEvbu3WsU4HXpekYefrp+l+c2E5Esyf48aT8/P1n1nsubN28ekpKSEBUVBZVKhZdeegnR0dHIzs5+rHXExMTgxo0bePPNN1FUVISRI0diwoQJFXrXlQkICED//v0NQVodZ2dn/POf/8Qrr7yCDh06IDw8HB999BGeffZZQ5smTZrg+PHjmDNnDqKiolBcXIzAwEAMHDjQcPjiww8/hFarxbhx45Cbm4vIyEjs378fHh4eAAA7Ozu88847SE5OhqOjI3r06IHt27fX4h2q3FdnU7Hl5E3kF5fByd4GL3QNxIhI/zrfDhFRTUnClAONlcjJyYGbmxuys7Ph6upqNK+oqAhJSUkIDg6GgwPPGbWEAQMGwMfHB1988YWlSzGrmn7Wrmfk4Y0vL0AIAU8ne9zNL4YkSVgxKoIDwIjI7KrL0PJk35OmhysoKMDq1asRFRUFpVKJbdu24dChQzh48KClS5MtXi2MiOoDhnQDIEkSvv32WyxevBjFxcUIDQ3F119/jf79+1u6NNkqf7UwfU+aVwsjIrlhSDcAjo6OOHTokKXLqFf0VwvbcvImUu4XGI5JsxdNRHLCkCarNSLSHx0CPXgvaCKSLbOGdA3HpBGZrLafseZezgxnIpIts5wnrb86VkFBgTlWT2Sgv7qb/qI3REQNiVl60kqlEu7u7oZrS6tUKpMurEH0KLRaLTIyMqBSqSq9axcRUX1ntm82Hx8fAHjojSOIakOhUCAgIIB/BBJRg2S2kJYkCb6+vvD29kZpaam5NkNWzs7OrsKNWIiIGgqz7yNUKpU8XkhERFQD7IIQERHJFEOaiIhIphjSREREMlXjY9L6i0jk5OTUWTFERETWQJ+dD7sgU41DOjc3FwDg78/77xIREdVEbm4u3Nzcqpxf4/tJa7Va/Pbbb3BxceE5qkRERI9ACIHc3Fw0adKk2tNIaxzSREREZF4cOEZERCRTDGkiIiKZYkgTERHJFEOaiIhIphjSREREMsWQJiIikimGNBERkUwxpImIiGSKIU1ERCRTDGkiIiKZYkgTERHJFEOaiIhIpv4f0NNEhdKrerAAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 600x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_vids = True\n",
    "\n",
    "method_names = [\"Original\", \"CoOp\", \"Name Tuning\"]\n",
    "method_markers = [\"o\", \"s\", \"^\"]\n",
    "method_text_embeds = [orig_text_sne, coop_text_sne, na_text_sne]\n",
    "\n",
    "color_cycler = iter(plt.cycler(\"color\", plt.cm.tab10.colors))\n",
    "class_colors = [next(color_cycler)[\"color\"] for _ in class_names]\n",
    "\n",
    "fig = plt.figure(figsize=(6, 4))\n",
    "\n",
    "# Plot videos\n",
    "if use_vids_in_sne and show_vids:\n",
    "    for color, video_embeds in zip(class_colors, support_video_sne):\n",
    "        plt.scatter(video_embeds[:, 0], video_embeds[:, 1], s=10, color=color, alpha=0.7)\n",
    "\n",
    "# Plot text for each method\n",
    "for name, marker, text_embeds in zip(method_names, method_markers, method_text_embeds):\n",
    "    for color, class_text_embed in zip(class_colors, text_embeds):\n",
    "        plt.scatter([class_text_embed[0]], [class_text_embed[1]], color=color, marker=marker, s=60)\n",
    "\n",
    "# Create legend\n",
    "legend_entries = [\n",
    "    mlines.Line2D([], [], marker=marker, label=name, color=\"black\", linestyle=\"None\")\n",
    "    for name, marker in zip(method_names, method_markers)\n",
    "]\n",
    "if use_vids_in_sne and show_vids:\n",
    "    legend_entries.append(mlines.Line2D([], [], label=\"Training Videos\", marker=\".\", color=\"black\", linestyle=\"None\"))\n",
    "plt.legend(handles=legend_entries)\n",
    "\n",
    "plt.axis(\"equal\")\n",
    "\n",
    "plt.title(\"Class Name Text Embeddings After Training\")\n",
    "plt.xticks([])\n",
    "plt.yticks([])\n",
    "plt.savefig(f\"text_embed_separation.method_markers.vids_{use_vids_in_sne and show_vids}.pdf\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAekAAAKVCAYAAADrxhOwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAubElEQVR4nO3df3TU9Z3v8dd3JpkJmfwgBCFmmQAJkvBbxNUWa3VFWtoe6Q9bFd26bW33Fq3b020rxeIiBYvI7d1tqWWvnl173Fara4uC9ti1qLu2pV1REHX5UTNAohG4CTGTTEgmyXzvH2MiMQRCfn3fM/N8nOPJYeabyTvWw7Pfz/eX47quKwAAYI7P6wEAAMCpEWkAAIwi0gAAGEWkAQAwikgDAGAUkQYAwCgiDQCAUUQaAACjiDQAAEYRacCYPXv26Itf/KKmTp2qnJwc5eXl6YILLtA999yj48ePn/Xnua6rhx56SFdccYWKiooUDAZVXl6uW265RbW1tSPwGwAYLg63BQXsuP/++3XzzTersrJSN998s2bOnKmOjg7t3LlT999/v+bNm6ctW7YM+PMSiYSuv/56PfLII1q2bJmuvfZaFRYWas+ePdq4caNaWlr05JNP6pJLLhnB3wrAYBFpwIgdO3bo0ksv1eLFi/X4448rGAz2ej8ej+vpp5/W0qVLB/yZ69ev1+233667775bK1as6PXe0aNHdfHFF6utrU379u3T2LFjh+PXADCMWO4GjPj+978vx3F033339Qm0JAUCgZ5AJxIJ3XPPPaqqqlIwGNSECRN044036s033+zZPh6Pa+PGjZoxY4Zuu+22Pp83ceJErV+/XkePHtW//Mu/9Lx++eWXa/bs2XrhhRf0gQ98QGPGjNFf/MVf6I477lBXV9cI/OYA+kOkAQO6urr07LPPasGCBQqHw2fcfvny5VqxYoUWL16srVu3au3atXr66ae1cOFC1dfXS5JeeuklNTY2aunSpXIc55Sfc9VVV8nn8+mZZ57p9fqRI0d03XXX6YYbbtATTzyhz372s1q3bp2+/vWvD/2XBTBgWV4PAECqr69Xa2urpk6desZt9+3bp/vuu08333yzNm3a1PP6/PnzdfHFF+sf//Efddddd6mmpkaSTvuZeXl5Ouecc3q27dbQ0KAnnniiZ8/9Ix/5iE6cOKHNmzfrtttuU1lZ2WB+TQBniT1pIMU899xzkqQvfOELvV6/6KKLNGPGDG3fvv2sPs913T572vn5+X2OfV9//fVKJBL6r//6r7MfGsCgEGnAgPHjxys3N1cHDx4847YNDQ2SpHPPPbfPe6WlpT3vd+/tnu4zY7GY6uvr+yyxT5w4sc+2JSUlvX4+gJFHpAED/H6/Fi1apJdeeqnXyV+nUlxcLEl6++23+7xXV1en8ePHS5IWLFigoqIibd26Vf1dxLF161YlEgktXry41+tHjx7ts+2RI0d6/XwAI49IA0asXLlSruvqK1/5iuLxeJ/3Ozo6tG3bNl1xxRWSpJ/97Ge93n/xxRe1d+9eLVq0SFLybPBvf/vb2rt3rzZu3Njn844dO6aVK1dq4sSJ+vKXv9zrvebmZm3durXXaw899JB8Pp8+/OEPD+n3BDBwnDgGGPHBD35Qmzdv1s0336wFCxZo+fLlmjVrljo6OrRr1y7dd999mj17trZs2aK//du/1aZNm+Tz+fSxj31Mhw4d0h133KFwOKxvfOMbPZ+5YsUKvfLKKz1f338zk+bmZj355JMqLCzsNUtxcbGWL1+umpoaTZ8+Xb/+9a91//33a/ny5Zw0BowmF4Apu3fvdv/mb/7GLSsrcwOBgBsKhdz58+e7//AP/+AeO3bMdV3X7erqcjds2OBOnz7dzc7OdsePH+/+9V//tVtbW9vn8xKJhPvzn//cvfzyy92xY8e6gUDAnTp1qrt8+XL38OHDfba/7LLL3FmzZrnPP/+8e+GFF7rBYNA999xz3dtvv93t6OgY8d8fwHu44xiAXi6//HLV19frtdde83oUIONxTBoAAKOINAAARrHcDQCAUexJAwBgFJEGAMAoIg0AgFGDvplJIpFQXV2d8vPz+30MHgAA6Mt1XTU3N6u0tFQ+X//7y4OOdF1d3YCeewsAAE6ttrZWkyZN6vf9QUc6Pz+/5wcUFBQM9mMAAMg40WhU4XC4p6X9GXSku5e4CwoKiDQAAINwpsPFnDgGAIBRRBoAAKOINAAARhFpAACMItIAABhFpAEAMIpIAwBgFJEGAMAoIo2zlojF1LZ3rxKxmNejAEBaG/Qdx5CZErGY6lbdoXikWoHyCpWuWytfKOT1WACQltiTxlmJ19QoHqmWXCkeqVa8ttbrkQAgbRFpnJVAWZkC5RWSIwUrpinAk9AAYMSw3I2z4guFVLpureK1tQqEwyx1A8AIItI4a75QSDlVVV6PAQBpj+VuAACMItIAABhFpAEAMIpIAwBgFJEGAMAoIg0AgFFEGgAAo4g0AABGEWkAAIwi0gAAGEWkAQAwikgDAGAUkQYAwCgiDQCAUUQaAACjiDQAAEYRaQAAjCLSAAAYRaQBADCKSAMAYBSRBgDAKCINAIBRRBoAAKOINAAARhHpYZCIxdS2d68SsZjXowAA0kiW1wOkukQsprpVdygeqVagvEKl69bKFwp5PRYAIA2wJz1E8ZoaxSPVkivFI9WK19Z6PRIAIE0Q6SEKlJUpUF4hOVKwYpoC4bDXIwEA0gTL3UPkC4VUum6t4rW1CoTDLHVb094iHY9I48qlYJ7X0wDAWSHSw8AXCimnqsrrMfB+7S3S1lul+v3S+Epp6SZCDSClsNyN9HU8kgy06ya/Nh70eiIAOCtEGulrXHlyD9pxpPFVUtFUrycCgLPCcjfSVzAvucTdeDAZaJa6AaQYIo30FsyTSuZ4PQUADArL3QAAGEWkAQAwikgDAGAUkQYAwCgiDQCAUUQaAACjiDQAAEYRaQAAjCLSAAAYRaQBADCKSAMAYBSRBgDAKCINAIBRRBoAAKOINAAARhFpAACMItIAABhFpAEAMIpIAwBgFJEGAMAoIg0AgFFEGgAAo4g0AABGEWkAAIwi0gAAGEWkAQAwikgDAGAUkQYAwCgiDQCAUUQaAACjiDQAAEYRaQAAjMryegAAQJo6+rq0d5vU1iTlFEozrpImzvJ6qpRCpAEAw6uhWtryVenN/5Ycv+T4JDchPb9eCl8sfWqzVFzh9ZQpgeVuAMDwaaiW7l8kvfVS8s9ul5ToSH6VpDd3Jt9vqPZuxhRCpAEAw2fLV6X26HtRfj+3K/n+48tHd64URaRP1t4ivb0n+TWVpOrcANLL0deTS9z9Bbqb2yXV/im5PU6LY9Ld2lukrbdK9ful8ZXS0k1SMM/rqc4sVecGkH72bksegz5TpKXkdnuf5ESyM2BPutvxSDJ0rpv82njQ64kGJlXnBpB+2pqSJ4kNhOOT2t4Z0XHSAZHuNq48uSfqONL4KqloqtcTDUyqzg0g/eQUJs/iHgg3IeWMHdFx0gHL3d2Cecml4saDydClypJxqs4NIP3MuCp5mdVAuF3J7XFa7EmfLJgnlcxJvdCl6twA0svEWdKki5LHm0/H8Sevl544c3TmSmFEGgAwfD79z1KwoP9QO/7k+5/aPLpzpSgiDQAYPsUV0le2S5MuTP7Z8Uu+7PeiPenC5PvccWxAOCYNABhexRXSTf/x7r27n0yexZ0z9t17d7PEfTaINABgZEycxXXQQ8RyNwAARhFpAACMItIAABhFpAEAMIpIAwBgFJEGAMAoIg0AgFFEGgAAo4g0AABGEWkAAIwi0gAAGEWkAQAwikgDAGAUkQYAwCgibU17i/T2nuRXAEBG43nSlrS3SFtvler3S+MrpaWbpGCe11MBADxCpC05HkkG2nWTXxsPSiVzvJ4KAMxo239Azc88o0RzVL78AuUvXqycyulejzViiLQl48qTe9D1+6XxVVLRVK8nAgAT4ocPq+47K3Vi1y7J75ccR3Jd1f/4xxpzwXyVrl+vwOTJXo857BzXdd3BfGM0GlVhYaGamppUUFAw3HNlrvaW5B500VSWugFAyUAfvOZaJVpapK6uvhv4/fLl5Wnqo4+kTKgH2lBOHLMmmJdc4ibQACBJqvvOyv4DLUldXUq0tKhu5e2jO9goINIAALPa9h9ILnH3F+huXV068fLLatt/YHQGGyVEGqeViMXUtnevErGY16MAyEDNzzyTPAY9EH6/mn/7zMgONMo4cQz9SsRiqlt1h+KRagXKK1S6bq18oZDXYwHIIInmaPIksYFwHCWi0ZEdaJSxJ41+xWtqFI9US64Uj1QrXlvr9UgAMowvvyB5WepAuK58aXYiM5FGvwJlZQqUV0iOFKyYpkA47PVIADJM/uLFZz4e3a2rK7l9GmG5G/3yhUIqXbdW8dpaBcJhlroBjLqcyukaM3++TuzZc/pY+/0aM2+ecqan141N2JPGaflCIeVUVRFoAJ4pvXu9fHl5/Z9A9u510qXrvz+6g40CIg0AMC0webKmPvqIxsybl3zB75eysnqiPWbevJS6kcnZYLkbAGBeYPJkTXno58l7d//2GSWiUfkK3r13d5otcZ+MSAMAUkZO5fS0fqDG+7HcDQCAUUQaAACjiDQAAEYRaQAAjCLSAAAYlTGRbu1o1b7j+9Ta0er1KAAADEhGXILV2tGq1X9YrUhTROWF5VqzcI1ys3O9HgsAgNPKiD3pmuYaRZoicl1XkaaIapt5mhMAwL6MiHRZfpnKC8vlOI7KC8sVzudpTgAA+zJiuTs3O1drFq5RbXOtwvlhlroBACkhIyItJUNdOa7S6zEAABiwjFjuBgAgFRHpYZCIxdS2d68SsZjXowAA0kjGLHePlEQsprpVdygeqVagvEKl69bKFwp5PRYAIA2wJz1E8ZoaxSPVkivFI9WK13J5FwBgeBDpIQqUlSlQXiE5UrBimgJhLu8CAAwPlruHyBcKqXTdWsVraxUIh1nqBgAMGyI9DHyhkHKqqrweAwCQZljuBgDAKCINAIBRRBoAAKOINAAARhFp9MLd0wDADs7uRg/ungYAtrAnjR7cPQ0AbCHS6MHd0wDAFpa70YO7pwGALUQavXD3NACwg+VuAACMItIAABhFpAEAMIpIAwBgFJEGAMAoIp2p2lukt/ckvwIATOISrEzU3iJtvVWq3y+Nr5SWbpKCeV5PBQB4H/akU1hrR6v2Hd+n1o7Ws/vG45FkoF03+bXx4MgMCAAYEvakU1RrR6tW/2G1Ik0RlReWa83CNcrNzh3YN48rT+5B1++XxldJRVNHdlgAwKAQ6RRV01yjSFNErusq0hRRbXOtKsdVDuybg3nJJe7Gg8lAs9QNACax3J2iyvLLVF5YLsdxVF5YrnD+WT4MI5gnlcwh0ABgGHvSKSo3O1drFq5RbXOtwvnhgS91AwBSBpFOYbnZuQNf4gYApByWuwEAMIpIAwBgFJEGAMAoIg0AgFFEGgAAo4g0AABGEWkAAIwi0gAAGEWkAQAwikgDAGCUqUgP+vnIAACkITP37h7S85EBAEhDZvakT/V8ZAAAMpmZSA/5+cgAAKQZM8vdPB8ZAIDezOxJS+89HzndAx1r79TrdU2KtXd6PQoAwDAze9KZItbeqRW/3KM3jrVo2oQ8bbh6rkJB/mcAAPRlak86ExxqiOmNYy1yXemNYy063MDlZgCAUyPSo2xKcUjTJuTJcaTzJuRpcnF6L+0DAAaPddZRFgpmacPVc3W4oVWTi3NZ6gYA9ItCeCAUzNLM0gKvxwAAGMdyNwAARhFp9MElYgBgA8vd6IVLxADADvak0QuXiAGAHUQavXCJGADYwTqmQa0drapprlFZftmo3yKVS8QAwA7+BjbGwnO1uUQMAGwg0sac6rnaleMqvR4LAEbdviNRPf3aEUVPdKpgTJaWzC5RVUlm7UAQaWO6n6vdvSfNc7UBZJpD9THd8u9P6c+xHfL526REjjqbZ+uffluiBZOL9IPPzdOU8SGvxxwVjuu67mC+MRqNqrCwUE1NTSooyKz/ZzPSWjtah/W52rH2Th1qiGlKceisjzF7eXwcQOb5w+F9+l9Pf1vKOSTX9UmuIzmuHCehztbJ6jhyjUK+Ej1xyyUpHeqBNpRIp7mhXPds4fg4gMxRE63RVb/6nLrUJsdJ9HnfdX1SIqi2w1/T+SXT9NjyhR5MOTwG2lAuwUpzQ7nu+VTHxwFgpHzjuRX9BlpS8nVfu7JLHtXOw43adyQ6yhOOPiKd5oZy3XP38XHHcTg+DmBEHWg8oAPvvNZvoLs5TkJZuYeVlXNEv3nt6ChN5x1OHEtzQ7nuOTc7V2sWrhnW4+MAcCrbD2+XI59cnT7SUnLZOyv/dTWd+OAoTOYtIp0BhnLdc252LpeAARhx0XhUjhwN7CQpR/KdUOGY7BGeynssdwMAPFcQKJCcgW7tKtGVoyWzS0ZyJBOINADAc4smL1LC7RrQto6TUGXeQlWW5I/wVN4j0gAAz00vmq7zzzlfPsd/2u1c1ye1TdGmz358lCbzFpEGAJhw14fuUl52qN9Qu65PfuXo/y7ZmNI3MjkbRBoAYEJZQZke/sTDmnfOXEmSz/HLJ7+cd1NVVTRL2z7z71o4ucrLMUcVZ3cDAMwoKyjTgx97UAcaD2h7zXZF26MqCBboyrIrdV7ReV6PN+qINADAnOlF0zW9aLrXY3iO5W4AAIwi0gAAGEWkAQAwikgDAGAUkQYAwCgiDQCAUUQaAACjiDQAAEYRaQAAjCLSAAAYRaQBADCKSAMAYBSRBgDAKCINAIBRRBoAAKOINAAARhFpAACMItIAABhFpAEAMIpIAwBgFJEGAMAoIg0AgFFEGgAAo4g0AABGEWkAAIwi0gAAGEWkAQAwikgDAGAUkQYAwCgiDQCAUUQaAACjiDQAAEYRaQAAjCLSAAAYRaQBADCKSAMAYBSRBgDAKCINAIBRRBoAAKOINAAARhFpAACMItIAABhFpAEAMIpIAwBgFJEGAMAoIg2zYu2der2uSbH2Tq9HAQBPZHk9AHAqsfZOrfjlHr1xrEXTJuRpw9VzFQrynyuAzMKeNEw61BDTG8da5LrSG8dadLih1euRAGDUEWmYNKU4pGkT8uQ40nkT8jS5ONfrkQBg1LF+CJNCwSxtuHquDje0anJxLkvdADISf/PBrFAwSzNLC7weAwA8w3I3AABGEWkAAIwi0gAAGEWkwU1DAMAoThzLcNw0BADsYk86w3HTEACwi0hnOG4aAgB2sa6Z4bhpCADYxd/I4KYhAGAUy90AABhFpAEAMIpIAwBgFJEGAMAoIg0AgFFEGgAAo4g0AABGEWkAAIwa9M1MXNeVJEWj0WEbBgCATNDdzu6W9mfQkW5ubpYkhcPhwX4EAAAZrbm5WYWFhf2+77hnyng/EomE6urqlJ+fL8dxBj0gAACZxnVdNTc3q7S0VD5f/0eeBx1pAAAwsjhxDAAAo4g0AABGEWkAAIwi0gAAGEWkAQAwikgDAGAUkQYAwCgiDQCAUUQaAACjiDQAAEYRaQAAjCLSAAAYRaQBADCKSAMAYBSRBgDAKCINAIBRRBoAAKOINAAARhFpAACMItIAABhFpAEAMIpIAwBgFJEGAMAoIg0AgFFEGgAAo4g0AABGEWkAAIwi0gAAGEWkAQAwikgDAGAUkQYAwCgiDQCAUUQaAACjiDQg6ac//akcx1FOTo4OHz7c5/3LL79cs2fP9mCywbvzzjvlOM4Z/7n88suH9eceOnRIjuPopz/96bB+LpCJsrweALCkvb1dq1at0r/92795PcqQffnLX9aSJUt6/vz222/rM5/5jG699VZdf/31Pa8XFBQM688999xztWPHDlVUVAzr5wKZiEgDJ1myZIkeeughfetb39K8efO8HmdIJk2apEmTJvX8+dChQ5KksrIyfeADHxixnxsMBkf084FMwnI3cJLbbrtNxcXFWrFixRm3vffee/XhD39YEyZMUCgU0pw5c3TPPfeoo6Oj13bdS+U7duzQwoULNWbMGE2ZMkUPPPCAJOmpp57SBRdcoNzcXM2ZM0dPP/10n5/15z//Wddff70mTJigYDCoGTNm6N577x3y79u9zN8d8G7PP/+8HMfR888/3+f3ePHFF3XppZcqNzdX5eXluvvuu5VIJHq2O9Vyd/fS++uvv65ly5apsLBQEydO1Je+9CU1NTX1+tnvvPOObrrpJo0bN055eXn6xCc+oUgkIsdxdOeddw75dwZSCXvSwEny8/O1atUqff3rX9ezzz6rK664ot9tq6urdf3112vq1KkKBAJ65ZVXdNddd2nfvn3613/9117bHjlyRF/84hd12223adKkSdq0aZO+9KUvqba2Vo899phuv/12FRYW6nvf+54+9alPKRKJqLS0VJL0P//zP1q4cKHKysr0gx/8QCUlJfrNb36jv/u7v1N9fb1Wr149ov9O3v973HDDDfrmN7+p1atXa8uWLVq5cqVKS0t14403nvH7r776al177bW66aab9Oqrr2rlypWS1PPvK5FI6KqrrtLOnTt155136oILLtCOHTt6LdsDGcUF4D7wwAOuJPfFF19029vb3fLycvfCCy90E4mE67que9lll7mzZs3q9/u7urrcjo4O98EHH3T9fr97/Pjxnvcuu+wyV5K7c+fOntcaGhpcv9/vjhkzxn3rrbd6Xt+9e7cryf3Rj37U89pHP/pRd9KkSW5TU1Ovn/m1r33NzcnJ6fWzTufgwYOuJHfjxo19fu+DBw/22va5555zJbnPPfdcn9/jT3/6U69tZ86c6X70ox/t83MeeOCBntdWr17tSnLvueeeXt978803uzk5OT3/np966ilXkrt58+Ze261fv96V5K5evXpAvyuQLljuBt4nEAho3bp12rlzpx599NF+t9u1a5eWLl2q4uJi+f1+ZWdn68Ybb1RXV5cOHDjQa9tzzz1XCxYs6PnzuHHjNGHCBJ1//vk9e8ySNGPGDEnqOcO8ra1N27dv16c//Wnl5uaqs7Oz55+Pf/zjamtr0x//+Mfh/PVPq6SkRBdddFGv1+bOnXvKM+JPZenSpX2+t62tTceOHZMk/ed//qck6Zprrum13bJlywY7MpDSiDRwCtddd50uuOACffe73+1zjFmSampqdOmll+qtt97SD3/4Q73wwgt68cUXe44Tnzhxotf248aN6/MZgUCgz+uBQEBSMs6S1NDQoM7OTm3atEnZ2dm9/vn4xz8uSaqvrx/6LzxAxcXFfV4LBoN9ft+Bfn8wGJT03r+vhoYGZWVl9fn3MnHixMGMC6Q8jkkDp+A4jjZs2KDFixfrvvvu6/P+448/rlgspl/96leaPHlyz+u7d+8e1jmKiork9/v1+c9/Xrfccsspt5k6deqgPz8nJ0dS8tKzk41m+E9WXFyszs5OHT9+vFeojxw54sk8gNfYkwb6ceWVV2rx4sX63ve+p5aWll7vOY4j6b09QUlyXVf333//sM6Qm5urv/qrv9KuXbs0d+5cXXjhhX3+OdXe7UBNmTJFkrRnz55er2/dunUoYw/aZZddJkl65JFHer3+i1/8wotxAM+xJw2cxoYNG7RgwQIdO3ZMs2bN6nl98eLFCgQCWrZsmW677Ta1tbVp8+bNamxsHPYZfvjDH+pDH/qQLr30Ui1fvlxTpkxRc3Oz3njjDW3btk3PPvvsoD/7L//yL1VZWalvfetb6uzsVFFRkbZs2aLf/e53w/gbDNySJUt0ySWX6Jvf/Kai0agWLFigHTt26MEHH5Qk+XzsVyCz8F88cBrz588/5UlLVVVV+uUvf6nGxsaeu3idf/75+tGPfjTsM8ycOVMvv/yyZs+erVWrVukjH/mIbrrpJj322GNatGjRkD7b7/dr27Ztqqqq0le/+lXdeOONCgaD+vGPfzxM058dn8+nbdu26brrrtPdd9+tT37yk3rhhRf0s5/9TJI0duxYT+YCvOK4rut6PQQAnM5DDz2kG264Qb///e+1cOFCr8cBRg2RBmDKww8/rLfeektz5syRz+fTH//4R23cuFHz58/vuUQLyBQckwZgSn5+vn7xi19o3bp1isViOvfcc/WFL3xB69at83o0YNSxJw0AgFGcOAYAgFFEGgAAo4g0AABGDfrEsUQiobq6OuXn5/fcfQkAAJyZ67pqbm5WaWnpaW/SM+hI19XVKRwOD/bbAQDIeLW1tZo0aVK/7w860vn5+T0/oKCgYLAfAwBAxolGowqHwz0t7c+gI929xF1QUECkAQAYhDMdLubEMQAAjCLSAAAYRaQBADCKSAMAYBSRBgDAKCINAIBRRBoAAKOINAAARhFpnLVELKa2vXuViMW8HgUA0tqg7ziGzJSIxVS36g7FI9UKlFeodN1a+UIhr8cCgLTEnjTOSrymRvFIteRK8Ui14rW1Xo8EAGmLSOOsBMrKFCivkBwpWDFNAZ6EBgAjhuVunBVfKKTSdWsVr61VIBxmqRsARhCRxlnzhULKqaryegwASHssdwMAYBSRBgDAKCINAIBRRBoAAKOINAAARhFpAACMItIAABhFpAEAMIpIAwBgFJEGAMAoIg0AgFFEGgAAo4g0AABGEWkAAIwi0gAAGEWkAQAwikgDAGAUkQYAwCgiDQCAUUQaAACjiDQAAEYRaQAAjCLSAAAYRaQBADCKSA+DRCymtr17lYjFvB4FAJBGsrweINUlYjHVrbpD8Ui1AuUVKl23Vr5QyOuxAABpgD3pIYrX1CgeqZZcKR6pVry21uuRAABpgkgPUaCsTIHyCsmRghXTFAiHvR4JAJAmWO4eIl8opNJ1axWvrVUgHGap25r2Ful4RBpXLgXzvJ4GAM4KkR4GvlBIOVVVXo+B92tvkbbeKtXvl8ZXSks3EWoAKYXlbqSv45FkoF03+bXxoNcTAcBZIdJIX+PKk3vQjiONr5KKpno9EQCcFZa7kb6Ceckl7saDyUCz1A0gxRBppLdgnlQyx+spAGBQWO4GAMAoIg0AgFFEGgAAo4g0AABGEWkAAIwi0gAAGEWkAQAwikgDAGAUkQYAwCgiDQCAUUQaAACjiDQAAEYRaQAAjCLSAAAYRaQBADCKSAMAYBSRBgDAKCINAIBRRBoAAKOINAAARhFpAACMItIAABhFpAEAMIpIAwBgFJEGAMCoLK8HAABkqKOvS3u3SW1NUk6hNOMqaeIsr6cyhUgDAEZXQ7W05avSm/8tOX7J8UluQnp+vRS+WPrUZqm4wuspTWC5GwAwehqqpfsXSW+9lPyz2yUlOpJfJenNncn3G6q9m9EQIg0AGD1bviq1R9+L8vu5Xcn3H18+unMZRaQBAKPj6OvJJe7+At3N7ZJq/5TcPsMRaQDA6Ni7LXkMeiAcv7T3yZGdJwVw4hgAYHS0Nb17ktgZ9qSl5HZt7/T9iP0H1PzMM0o0R+XLL1D+4sXKqZw+/LMaQaQBAKMjpzB5FvdAuAkpZ2zPH+OHD6vuOyt1Ytcuye+XHEdyXdX/+Mcac8F8la5fr8DkySMzt4dY7gYAjI4ZVw1sL1pKbjfjKknJQB+85lqd2LMn+V5Xl9TZmfwq6cQre3TwmmsVP3x4JKb2FJEGAIyOibOkSRed+bi0409eLz1xpiSp7jsrlWhp6YlyH11dSrS0qG7l7cM8sPeINABg9Hz6n6VgQf+hdvzJ9z+1WVLyGPSJXbv6D3S3ri6dePllte0/MMwDe4tIn6y9RXp7T/JrKknVuQFknuIK6SvbpUkXJv/s+CVf9nvRnnRh8v137zjW/MwzyWPQA+H3q/m3z4zA0N7hxLFu7S3S1lul+v3S+Epp6SYpmOf1VGeWqnMDyFzFFdJN//HuvbufTJ7FnTP23Xt3z+y1aaI5mjxJbCAcR4lodNjH9RKR7nY8kgyd6ya/Nh6USuZ4PdWZpercADBx1hkfqOHLL0j+/TYQritfQcEwDGYHy93dxpUn90QdRxpfJRVN9XqigUnVuQFgAPIXLz7z8ehuXV3J7dMIe9LdgnnJpeLGg8nQpcqScarODQADkFM5XWPmz09efnW6WPv9GjNvnnKmp9eNTdiTPlkwL7lUnGqhS9W5AWAASu9eL19eXv8nkPn98uXlqXT990d3sFFApAEApgUmT9bURx/RmHnzki/4/VJWVk+0x8ybp6mPPpKWdxxjuRsAYF5g8mRNeejnyXt3//YZJaJR+QrevXd3mi1xn4xIAwBSRk7l9LR+oMb7sdwNAIBRRBoAAKOINAAARhFpAACMItIAABhFpAEAMIpIAwBgFJEGAMAoIg0AgFFEGgAAo4g0AABGEWkAAIwi0gAAGEWkAQAwikgDAGAUkbamvUV6e0/yKwAgo2V5PQBO0t4ibb1Vqt8vja+Ulm6SgnleTwUA8Ah70pYcjyQD7brJr40HvZ4IAOAhIm3JuPLkHrTjSOOrpKKpXk8EAPAQy92WBPOSS9yNB5OBZqkbADIakbYmmCeVzPF6CgCAASx3AwBgFJHGaSViMbXt3atELOb1KACQcVjuRr8SsZjqVt2heKRagfIKla5bK18o5PVYAJAx2JNGv+I1NYpHqiVXikeqFa+t9XokAMgoRBr9CpSVKVBeITlSsGKaAuGw1yMBQEZhuRv98oVCKl23VvHaWgXCYZa6AWCUEWmcli8UUk5VlddjAEBGYrkbAACjiDQAAEYRaQAAjCLSAAAYRaQBADCKSAMAYBSRBgDAKCINAIBRGRPp1o5W7Tu+T60drV6PAgDAgGTEHcdaO1q1+g+rFWmKqLywXGsWrlFudq7XYwEAcFoZsSdd01yjSFNErusq0hRRbTNPcwIA2JcRkS7LL1N5Ybkcx1F5YbnC+TzNCQBgX0Ysd+dm52rNwjWqba5VOD/MUjcAICVkRKSlZKgrx1V6PQYAAAOWEcvdAACkIiI9DBKxmNr27lUiFvN6FABAGsmY5e6RkojFVLfqDsUj1QqUV6h03Vr5QiGvxwIApAH2pIcoXlOjeKRacqV4pFrxWi7vAgAMDyI9RIGyMgXKKyRHClZMUyDM5V0AgOHBcvcQ+UIhla5bq3htrQLhMEvdAIBhQ6SHgS8UUk5VlddjAADSDMvdAAAYRaQBADCKSAMAYBSRBgDAKCKNXrh7GgDYwdnd6MHd0wDAFvak0YO7pwGALUQaPbh7GgDYwnI3enD3NACwhUijF+6eBgB2sNwNAIBRRBoAAKOINAAARhFpAACMItIAABhFpDNVe4v09p7kVwCASVyClYnaW6Stt0r1+6XxldLSTVIwz+upAADvw550CmvtaNW+4/vU2tF6dt94PJIMtOsmvzYeHJkBAQBDwp50imrtaNXqP6xWpCmi8sJyrVm4RrnZuQP75nHlyT3o+v3S+CqpaOrIDgsAGBQinaJqmmsUaYrIdV1FmiKqba5V5bjKgX1zMC+5xN14MBlolroBwCSWu1NUWX6ZygvL5TiOygvLFc4/y4dhBPOkkjkEGgAMY086ReVm52rNwjWqba5VOD888KVuAEDKINIpLDc7d+BL3ACAlMNyNwAARhFpAACMItIAABhFpAEAMIpIAwBgFJEGAMAoIg0AgFFEGgAAo4g0AABGEWkAAIwyFelBPx8ZAIA0ZObe3UN6PjIAACPgQOMBbT+8XdF4VAWBAi2avEjTi6aP2s83E+khPR8ZAIBhVBOt0Xd/913t/n+75Xf8chxHruvqJ6/8ROefc77u+tBdKisoG/E5zCx3D/n5yAAADIOaaI2WPbVMr9a/KknqcrvUmehUl9slSXq1/lUte2qZaqI1Iz6L47quO5hvjEajKiwsVFNTkwoKCoZlmNaOVp6PDADw1Od//Xm9Wv9qT5RPxe/4NfecuXrwYw8O6mcMtKFm9qSl956PnO6BjrV36vW6JsXaO70eBQBwkgONB7T7/+0+baCl5N71rmO7dKDxwIjOY+aYdKaItXdqxS/36I1jLZo2IU8brp6rUJD/GQDAgu2Ht8vv+M8YaSm5N729ZvuInkhmak86ExxqiOmNYy1yXemNYy063MDlZgBgRTQeleM4A9rWcRxF26MjOg+RHmVTikOaNiFPjiOdNyFPk4vTe2kfAFJJQaBAAz1Vy3VdFQSH55ys/rDOOspCwSxtuHquDje0anJxLkvdAGDIosmL9JNXfjKgbbvcLl1ZduWIzsOetAdCwSzNLC0g0ABgzPSi6Tr/nPPld/yn3c7v+DV/wnydV3TeiM5DpAEAOMldH7pLoexQv6H2O36FskNad8m6EZ+FSKMPLhEDkMnKCsr08Cce1txz5kpKRjnLl9UT7bnnzNXDn3h4VO44xnoreuESMQBIhvrBjz2YvHd3zXZF26MqCBboyrIrR3yJ+2T87YteTnWJ2MzSkT17EQCsml40fVQfqPF+LHejFy4RAwA72JM2qLWjVTXNNSrLLxv1W6RyiRgA2MHfwMZYeK529yViAABvsdxtzKmeqw0AyExE2hieqw0A6MZytzG52blas3DNsD5XO9beqUMNMU0pDp31MWYvj48DQKYj0gZ1P1d7OAzlumcLx8cBIJOx3J3mhvJoTI6PA4C3iHSaG8p1zxwfBwBvOe5AH5z5PtFoVIWFhWpqalJBAZfrWBZr7xz0dc+tHa3DenwcADDwhnJMOgMM5brn4Tw+DgA4Oyx3AwBgFJEGAMAoIg0AgFFEGgAAo4g0AABGEWkAAIwi0gAAGEWkAQAwikgDAGAUdxwDAKSlfUeievq1I4qe6FTBmCwtmV2iqpLUuo01kQYApJVD9TH9/aO79XLNO/L7HPkcKeFK//TbP2vB5CL94HPzNGV8yOsxB4TlbgBA2jhUH9Mn7/29XnmzSZLUlXDV0eWqK5F8ltTu2nf0yXt/r0P1MS/HHDAiDQBIG3//6G61tHf2RPn9uhKuWto79a1/f2WUJxscIg0ASAv7jkT1cs07/Qa6W1fC1c7Djdp3JDpKkw0ekQYApIWnXzsiv88Z0LZ+n6PfvHZ0hCcaOiINAEgL0ROdGmCj5XOkphMdIzvQMCDSAIC0UDAmS2dY6e6RcKXCMdkjO9AwINIAgLSwZHbJGY9Hd+tKuFoyu2SEJxo6Ig0ASAtVJQW6oGzsGY9L+32OLpxcpMqS/FGabPCINAAgbfyfa85XXjCr31D7fY7ygln635+bN8qTDQ6RBgCkjSnjQ3rilks0PzxWUjLK2X6nJ9rzw2P1xC2XpMwdx7gtKAAgrUwZH9Jjyxdq35GofvPaUTWd6FDhmGwtmV2SEkvcJyPSAIC0VFVSkHIP1Hg/lrsBADCKSAMAYBSRBgDAKCINAIBRRBoAAKOINAAARhFpAACMItIAABhFpAEAMIpIAwBgFJEGAMAoIg0AgFFEGgAAo4g0AABGEWkAAIwi0gAAGEWkAQAwikgDAGAUkQYAwCgiDQCAUUQaAACjiDQAAEYRaQAAjCLSAAAYRaQBADCKSAMAYBSRBgDAKCINAIBRRBoAAKOINMyKtXfq9bomxdo7vR4FADyR5fUAwKnE2ju14pd79MaxFk2bkKcNV89VKMh/rgAyC3vSMOlQQ0xvHGuR60pvHGvR4YZWr0cCgFFHpGHSlOKQpk3Ik+NI503I0+TiXK9HAoBRx/ohTAoFs7Th6rk63NCqycW5LHUDyEj8zQezQsEszSwt8HoMAPAMy90AABhFpAEAMIpIAwBgFJEGNw0BAKM4cSzDcdMQALCLPekMx01DAMAuIp3huGkIANjFumaG46YhAGAXfyODm4YAgFEsdwMAYBSRBgDAKCINAIBRRBoAAKOINAAARhFpAACMItIAABhFpAEAMGrQNzNxXVeSFI1Gh20YAAAyQXc7u1van0FHurm5WZIUDocH+xEAAGS05uZmFRYW9vu+454p4/1IJBKqq6tTfn6+HMcZ9IAAAGQa13XV3Nys0tJS+Xz9H3kedKQBAMDI4sQxAACMItIAABhFpAEAMIpIAwBgFJEGAMAoIg0AgFFEGgAAo4g0AABGEWkAAIwi0gAAGEWkAQAwikgDAGDU/wcsT/5Pw5tHxQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 600x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_vids = True\n",
    "\n",
    "method_names = [\"CoOp\", \"Name Tuning\"]\n",
    "method_text_embeds = [coop_text_sne, na_text_sne]\n",
    "\n",
    "color_cycler = iter(plt.cycler(\"color\", plt.cm.tab10.colors))\n",
    "class_colors = [next(color_cycler)[\"color\"] for _ in class_names]\n",
    "\n",
    "fig,axs = plt.subplots(2, 1, figsize=(6, 8), sharex=True, sharey=True)\n",
    "\n",
    "\n",
    "# Plot text for each method\n",
    "for i, (name, text_embeds) in enumerate(zip(method_names, method_text_embeds)):\n",
    "    ax = axs[i]\n",
    "    \n",
    "    if use_vids_in_sne and show_vids:\n",
    "        for color, video_embeds, class_text_embed in zip(class_colors, support_video_sne, text_embeds):\n",
    "            ax.scatter(video_embeds[:, 0], video_embeds[:, 1], s=10, color=color, marker=\".\", alpha=0.7)\n",
    "            ax.scatter([class_text_embed[0]], [class_text_embed[1]], color=color, marker=\"o\")\n",
    "    else:\n",
    "        for color, class_text_embed in zip(class_colors, text_embeds):\n",
    "            ax.scatter([class_text_embed[0]], [class_text_embed[1]], color=color, marker=\"o\")\n",
    "            \n",
    "    ax.set_title(name)\n",
    "    ax.set_xticks([])\n",
    "    ax.set_yticks([])\n",
    "\n",
    "#plt.axis(\"equal\")\n",
    "\n",
    "plt.xticks([])\n",
    "plt.yticks([])\n",
    "plt.savefig(f\"text_embed_separation.method_markers.vids_{use_vids_in_sne and show_vids}.pdf\")\n",
    "plt.show()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Average Similarity between Class Embeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original: 1.000\n",
      "CoOp: 1.000\n",
      "Name Tuning: 1.000\n",
      "Original: 0.329\n",
      "CoOp: 0.363\n",
      "Name Tuning: 0.434\n"
     ]
    }
   ],
   "source": [
    "for name, text_embeds in zip(method_names, method_text_embeds):\n",
    "    text_embeds = torch.from_numpy(text_embeds)\n",
    "    similarities = F.cosine_similarity(text_embeds.unsqueeze(0).expand(n_way, n_way, -1), text_embeds.unsqueeze(1).expand(n_way, n_way, -1), dim=-1)\n",
    "    similarities *= (1 - torch.eye(n_way))\n",
    "    mean_class_similarity = similarities.sum(dim=(0,1)) / (n_way * (n_way - 1))\n",
    "    print(f\"{name}: {mean_class_similarity:.3f}\")\n",
    "    \n",
    "for name, text_embeds in zip(method_names, method_text_embeds):\n",
    "    text_embeds = torch.from_numpy(text_embeds)\n",
    "    distances = torch.norm(text_embeds.unsqueeze(0) - text_embeds.unsqueeze(1), dim=-1)\n",
    "    distances *= (1 - torch.eye(n_way))\n",
    "    mean_class_distance = distances.sum(dim=(0,1)) / (n_way * (n_way - 1))\n",
    "    print(f\"{name}: {mean_class_distance:.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'classifier' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[42], line 3\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[39m# Get tuned class name embeddings\u001b[39;00m\n\u001b[0;32m      2\u001b[0m orig_name_embeds, _ \u001b[39m=\u001b[39m vlm\u001b[39m.\u001b[39mget_input_word_embeddings(ALLOWED_CLASSES)\n\u001b[1;32m----> 3\u001b[0m tuned_name_embeds, attn_mask \u001b[39m=\u001b[39m classifier\u001b[39m.\u001b[39mtuned_input_embeds[\u001b[39m0\u001b[39m]\n\u001b[0;32m      4\u001b[0m tuned_name_offset \u001b[39m=\u001b[39m tuned_name_embeds \u001b[39m-\u001b[39m orig_name_embeds\n",
      "\u001b[1;31mNameError\u001b[0m: name 'classifier' is not defined"
     ]
    }
   ],
   "source": [
    "# Get tuned class name embeddings\n",
    "orig_name_embeds, _ = vlm.get_input_word_embeddings(ALLOWED_CLASSES)\n",
    "tuned_name_embeds, attn_mask = classifier.tuned_input_embeds[0]\n",
    "tuned_name_offset = tuned_name_embeds - orig_name_embeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original name embed norms: 0.3848233222961426\n",
      "Tuned name embed norms: 0.8528121709823608\n",
      "Tuned name offset norms: 0.5832348465919495\n",
      "\n",
      "Offset norms per class:\n",
      " -  dancing ballet\n",
      "    0.84 1.00\n",
      " -  dancing charleston\n",
      "    1.23 1.55\n",
      " -  dancing macarena\n",
      "    1.18 1.94 1.00\n"
     ]
    }
   ],
   "source": [
    "print(\"Original name embed norms:\", torch.norm(orig_name_embeds, dim=-1).mean().item())\n",
    "print(\"Tuned name embed norms:\", torch.norm(tuned_name_embeds, dim=-1).mean().item())\n",
    "print(\"Tuned name offset norms:\", torch.norm(tuned_name_offset, dim=-1).mean().item())\n",
    "\n",
    "print(\"\\nOffset norms per class:\")\n",
    "offset_norms = torch.norm(tuned_name_offset, dim=-1)\n",
    "for category_ind in range(n_way):\n",
    "    start_token_ind = vlm.text_start_special_token_count()\n",
    "    end_token_ind = attn_mask[category_ind].sum() - vlm.text_end_special_token_count()\n",
    "    print(\" - \", class_names[category_ind])\n",
    "    print(\"   \", \" \".join([f\"{norm:.2f}\" for norm in offset_norms[category_ind, start_token_ind:end_token_ind]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Original name token similarity:\n",
      " -  1.00 0.01 0.31\n",
      "\n",
      "Tuned name token similarity:\n",
      " -  0.05 -0.05 0.32\n",
      "\n",
      "Tuned name offset token similarity:\n",
      " -  -0.02 -0.06 0.00\n"
     ]
    }
   ],
   "source": [
    "def print_similarity_per_token(embed):\n",
    "    similarity = F.cosine_similarity(\n",
    "        embed.unsqueeze(0).expand(n_way, n_way, *embed.shape[1:]),\n",
    "        embed.unsqueeze(1).expand(n_way, n_way, *embed.shape[1:]),\n",
    "        dim=-1\n",
    "    ).cpu()\n",
    "    similarity *= (1 - torch.eye(n_way).unsqueeze(2))\n",
    "    similarity = similarity.sum(dim=(0, 1)) / (n_way * (n_way - 1))\n",
    "    similarity = similarity[1:-1]\n",
    "    print(\" - \", \" \".join([f\"{s.item():.2f}\" for s in similarity]))\n",
    "\n",
    "print(\"\\nOriginal name token similarity:\")\n",
    "print_similarity_per_token(orig_name_embeds)\n",
    "\n",
    "print(\"\\nTuned name token similarity:\")\n",
    "print_similarity_per_token(tuned_name_embeds)\n",
    "\n",
    "print(\"\\nTuned name offset token similarity:\")\n",
    "print_similarity_per_token(tuned_name_offset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dancing  ballet \n",
      "dance  recital \n",
      "belle erts \n",
      "legacy  bly\n",
      "\n",
      "\n",
      "dancing  charleston \n",
      "madhuridixit  tooting \n",
      "sleet  memph\n",
      "poles  telford \n",
      "\n",
      "\n",
      "punjab <|startoftext|> arena \n",
      "luxury >  pier \n",
      "tamil  that  stadium \n",
      "dancing  for  fountain \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def closest_words(word_embedding: torch.Tensor, k=1):\n",
    "    distances = torch.norm(vlm.model.text_model.embeddings.token_embedding.weight - word_embedding.unsqueeze(0), dim=1)\n",
    "    \n",
    "    best = torch.topk(distances, k, largest=False)\n",
    "    distances = best.values.cpu().tolist()\n",
    "    tokens = best.indices\n",
    "    words = [vlm.tokenizer.decode(t) for t in tokens]\n",
    "    \n",
    "    return words, distances\n",
    "\n",
    "k = 4\n",
    "for category_ind in range(n_way):\n",
    "    n_tokens = attn_mask[category_ind].sum()\n",
    "    start_token_ind = vlm.text_start_special_token_count()\n",
    "    end_token_ind = n_tokens - vlm.text_end_special_token_count()\n",
    "    \n",
    "    # Print a line for most similar replacements, then next most similar, etc\n",
    "    lines = [[] for _ in range(k)]\n",
    "    for token_ind in range(start_token_ind, end_token_ind):\n",
    "        words, distances = closest_words(tuned_name_embeds[category_ind, token_ind], k)\n",
    "        for i in range(k):\n",
    "            #lines[i] += [f\"{words[i]} ({distances[i]:.3f})\"]\n",
    "            lines[i] += [words[i]]\n",
    "    for line in lines:\n",
    "        print(\" \".join(line))\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dancing  (0.000, 0.000) ballet  (0.000, 0.000)\n",
      "dance  (0.259, 0.019) 2 (0.298, -0.041)\n",
      "coscino  (0.290, -0.002) 0 (0.298, -0.041)\n",
      "0 (0.291, -0.002) 1 (0.298, -0.041)\n",
      "\n",
      "\n",
      "dancing  (0.000, 0.000) charleston  (0.000, 0.000)\n",
      "dance  (0.259, -0.008) 2 (0.301, -0.007)\n",
      "coscino  (0.290, -0.011) 0 (0.301, -0.007)\n",
      "0 (0.290, -0.011) 1 (0.301, -0.007)\n",
      "\n",
      "\n",
      "dancing  (0.000, 0.000) mac (0.000, 0.000) arena  (0.000, 0.000)\n",
      "dance  (0.260, 0.005) 2 (0.272, 0.005) 2 (0.291, -0.018)\n",
      "coscino  (0.290, 0.015) 0 (0.272, 0.005) 0 (0.291, -0.018)\n",
      "0 (0.290, 0.013) 1 (0.272, 0.005) 1 (0.291, -0.018)\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def closest_words_along_line(base_embed: torch.Tensor, direction_embed: torch.Tensor, k=1):\n",
    "    token_embeds = vlm.model.text_model.embeddings.token_embedding.weight\n",
    "    rel_token_embeds = token_embeds - base_embed\n",
    "    rel_token_embeds_parallel_component = rel_token_embeds @ F.normalize(direction_embed, dim=0)\n",
    "    rel_token_embeds_along_dir = F.normalize(direction_embed, dim=0).unsqueeze(0) * rel_token_embeds_parallel_component.unsqueeze(1)\n",
    "    rel_token_embeds_perpendicular = rel_token_embeds - rel_token_embeds_along_dir\n",
    "    \n",
    "    distances = torch.norm(rel_token_embeds_perpendicular, dim=1)\n",
    "    \n",
    "    # Rule out matches that are in opposite direction to direction_embed\n",
    "    # Also rule out matches that are very close to base_embed\n",
    "    #distances[rel_token_embeds_parallel_component <= 0.5] += 1000\n",
    "    \n",
    "    best = torch.topk(distances, k, largest=False)\n",
    "    distances = best.values.cpu().tolist()\n",
    "    tokens = best.indices\n",
    "    words = [vlm.tokenizer.decode(t) for t in tokens]\n",
    "    \n",
    "    parallel_multiplier = (rel_token_embeds_parallel_component[tokens] / torch.norm(direction_embed, dim=0)).tolist()\n",
    "    \n",
    "    return words, distances, parallel_multiplier\n",
    "\n",
    "k = 4\n",
    "for category_ind in range(n_way):\n",
    "    n_tokens = attn_mask[category_ind].sum()\n",
    "    start_token_ind = vlm.text_start_special_token_count()\n",
    "    end_token_ind = n_tokens - vlm.text_end_special_token_count()\n",
    "    \n",
    "    # Print a line for most similar replacements, then next most similar, etc\n",
    "    lines = [[] for _ in range(k)]\n",
    "    for token_ind in range(start_token_ind, end_token_ind):\n",
    "        words, distances, parallel_multiplier = closest_words_along_line(orig_name_embeds[category_ind, token_ind], tuned_name_offset[category_ind, token_ind], k)\n",
    "        for i in range(k):\n",
    "            lines[i] += [f\"{words[i]} ({distances[i]:.3f}, {parallel_multiplier[i]:.3f})\"]\n",
    "    for line in lines:\n",
    "        print(\" \".join(line))\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "vlm_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}