experiment_analysis/head_classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, '../src')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading region bounding boxes for computing carbon emissions region, this may take a moment...\n",
      " 454/454... rate=462.48 Hz, eta=0:00:00, total=0:00:00, wall=08:21 CETT\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "import classify_attention_patterns\n",
    "import classify_normed_patterns\n",
    "from argparse import Namespace\n",
    "from run_glue import load_and_cache_examples, set_seed\n",
    "from model_bert import BertForSequenceClassification, BertForMaskedLM\n",
    "from config_bert import BertConfig\n",
    "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset\n",
    "from transformers import BertTokenizer\n",
    "import numpy as np\n",
    "import torch\n",
    "import random\n",
    "from collections import Counter\n",
    "\n",
    "def set_seed(seed):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "head_classifier_model, label2id, min_max_size = classify_attention_patterns.load_model(\"../models/head_classifier/classify_attention_patters.tar\")\n",
    "head_classifier_model = head_classifier_model.eval().cuda()\n",
    "id2label = {idx:label for label, idx in label2id.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "normed_head_classifier_model, normed_label2id, normed_min_max_size = classify_normed_patterns.load_model(\"../models/head_classifier/classify_normed_patterns.tar\")\n",
    "normed_head_classifier_model = normed_head_classifier_model.eval().cuda()\n",
    "normed_id2label = {idx:label for label, idx in normed_label2id.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fine-tuned model - Super Survivors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CoLA\n",
      "attention: {'mix': 0.4218823529411765, 'other': 0.4046470588235294, 'diagonal': 0.1458235294117647, 'vertical': 0.02123529411764706, 'block': 0.006411764705882353}\n",
      "weight normed: {'Diagonal': 0.42823529411764705, 'Heterogeneous': 0.3382941176470588, 'Vertical + Diagonal': 0.10252941176470588, 'Block': 0.09682352941176471, 'Vertical': 0.03411764705882353}\n",
      "SST-2\n",
      "attention: {'other': 0.3487368421052632, 'diagonal': 0.25705263157894737, 'block': 0.1965263157894737, 'mix': 0.19473684210526315, 'vertical': 0.0029473684210526317}\n",
      "weight normed: {'Block': 0.36326315789473684, 'Heterogeneous': 0.26063157894736844, 'Diagonal': 0.18894736842105264, 'Vertical + Diagonal': 0.17410526315789474, 'Vertical': 0.013052631578947368}\n",
      "MRPC\n",
      "attention: {'vertical': 0.377, 'block': 0.23685714285714285, 'other': 0.18492857142857144, 'diagonal': 0.10107142857142858, 'mix': 0.10014285714285714}\n",
      "weight normed: {'Heterogeneous': 0.2807857142857143, 'Vertical': 0.25007142857142856, 'Vertical + Diagonal': 0.23157142857142857, 'Diagonal': 0.18985714285714286, 'Block': 0.047714285714285716}\n",
      "STS-B\n",
      "attention: {'other': 0.6768333333333333, 'block': 0.13475, 'diagonal': 0.1115, 'mix': 0.059583333333333335, 'vertical': 0.017333333333333333}\n",
      "weight normed: {'Heterogeneous': 0.438, 'Block': 0.28391666666666665, 'Diagonal': 0.1285, 'Vertical + Diagonal': 0.12166666666666667, 'Vertical': 0.027916666666666666}\n",
      "QQP\n",
      "attention: {'other': 0.5302608695652173, 'mix': 0.14478260869565218, 'diagonal': 0.1376521739130435, 'vertical': 0.13460869565217393, 'block': 0.052695652173913046}\n",
      "weight normed: {'Vertical': 0.34956521739130436, 'Vertical + Diagonal': 0.1988695652173913, 'Heterogeneous': 0.198, 'Diagonal': 0.16643478260869565, 'Block': 0.08713043478260869}\n",
      "MNLI\n",
      "attention: {'other': 0.354, 'diagonal': 0.2806, 'block': 0.2482, 'mix': 0.0948, 'vertical': 0.0224}\n",
      "weight normed: {'Diagonal': 0.3262, 'Block': 0.2945, 'Heterogeneous': 0.2137, 'Vertical + Diagonal': 0.1589, 'Vertical': 0.0067}\n",
      "QNLI\n",
      "attention: {'other': 0.5126451612903226, 'diagonal': 0.2372258064516129, 'block': 0.16548387096774195, 'mix': 0.05987096774193548, 'vertical': 0.024774193548387096}\n",
      "weight normed: {'Diagonal': 0.36316129032258065, 'Heterogeneous': 0.2718709677419355, 'Vertical + Diagonal': 0.16612903225806452, 'Block': 0.10541935483870968, 'Vertical': 0.09341935483870968}\n",
      "RTE\n",
      "attention: {'block': 0.5658181818181818, 'diagonal': 0.158, 'other': 0.15454545454545454, 'mix': 0.09945454545454545, 'vertical': 0.02218181818181818}\n",
      "weight normed: {'Heterogeneous': 0.46036363636363636, 'Vertical + Diagonal': 0.22927272727272727, 'Block': 0.1698181818181818, 'Diagonal': 0.1390909090909091, 'Vertical': 0.0014545454545454545}\n"
     ]
    }
   ],
   "source": [
    "set_seed(1337)\n",
    "for task in [\"CoLA\", \"SST-2\", \"MRPC\", \"STS-B\", \"QQP\", \"MNLI\", \"QNLI\", \"RTE\"]:\n",
    "    attention_counter = Counter()\n",
    "    normed_attention_counter = Counter()\n",
    "    for seed in [\"seed_1337\", \"seed_42\", \"seed_86\", \"seed_71\", \"seed_166\"]:\n",
    "        #Load Model\n",
    "        model_path = f\"../models/finetuned/{task}/{seed}/\"\n",
    "        tokenizer = BertTokenizer.from_pretrained(model_path)\n",
    "        config = BertConfig.from_pretrained(model_path)\n",
    "        config.output_attentions = True\n",
    "        transformer_model = BertForSequenceClassification.from_pretrained(model_path,  config=config)\n",
    "        # Prune\n",
    "        mask_path = f\"../masks/heads_mlps_super/{task}/{seed}/\"\n",
    "        head_mask = np.load(f\"{mask_path}/head_mask.npy\")\n",
    "        mlp_mask = np.load(f\"{mask_path}/mlp_mask.npy\")\n",
    "        head_mask = torch.from_numpy(head_mask)\n",
    "        heads_to_prune = {} \n",
    "        for layer in range(len(head_mask)):\n",
    "            heads_to_mask = [h[0] for h in (1 - head_mask[layer].long()).nonzero().tolist()]\n",
    "            heads_to_prune[layer] = heads_to_mask\n",
    "        mlps_to_prune = [h[0] for h in (1 - torch.from_numpy(mlp_mask).long()).nonzero().tolist()]\n",
    "\n",
    "        transformer_model.prune_heads(heads_to_prune)\n",
    "        transformer_model.prune_mlps(mlps_to_prune)\n",
    "        transformer_model = transformer_model.eval()\n",
    "        transformer_model.cuda()\n",
    "        args = Namespace(data_dir=f\"../data/glue/{task}/\", local_rank=-1, \n",
    "                         model_name_or_path=model_path, \n",
    "                         overwrite_cache=False, model_type=\"bert\", max_seq_length=128)\n",
    "        eval_dataset = load_and_cache_examples(args, task.lower(), tokenizer, evaluate=True)\n",
    "        eval_sampler = RandomSampler(eval_dataset)\n",
    "        eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=1)\n",
    "        input_data = None\n",
    "        attention_head_types = [[] for _ in range(12)]\n",
    "        normed_attention_head_types = [[] for _ in range(12)]\n",
    "\n",
    "        k = 0\n",
    "        for batch in eval_dataloader:\n",
    "            batch = tuple(t.to(\"cuda:0\") for t in batch)\n",
    "            n_tokens = batch[1].sum()\n",
    "            if n_tokens < min_max_size[0]:\n",
    "                continue\n",
    "            input_data =  {\"input_ids\": batch[0], \"attention_mask\": batch[1], \"labels\": batch[3]}\n",
    "            with torch.no_grad():\n",
    "                _, _, attentions, allalpha_f, attention_mask = transformer_model(**input_data)\n",
    "                for layer in range(len(attentions)):\n",
    "                    if attentions[layer] is None:\n",
    "                        continue\n",
    "\n",
    "                    num_classifier_tokens = max(n_tokens, min_max_size[0]+20)\n",
    "                    head_attentions = attentions[layer].transpose(0, 1)\n",
    "\n",
    "                    head_attentions = head_attentions[:,:,:num_classifier_tokens,:num_classifier_tokens]\n",
    "                    logits = head_classifier_model(head_attentions)\n",
    "                    label_ids = torch.argmax(logits, dim=-1)\n",
    "                    labels = [id2label[int(label_id.item())] for label_id in label_ids]\n",
    "                    if len(attention_head_types[layer]) == 0:\n",
    "                        for i in range(len(labels)):\n",
    "                            attention_head_types[layer].append(Counter())\n",
    "                    for i, label in enumerate(labels):\n",
    "                        attention_head_types[layer][i][label] += 1\n",
    "                    \n",
    "                    num_classifier_tokens = max(n_tokens, normed_min_max_size[0]+20)\n",
    "                    normed_alpha_f = allalpha_f[layer].norm(dim=-1)\n",
    "                    normed_head_attentions = normed_alpha_f.transpose(0, 1)\n",
    "\n",
    "                    normed_head_attentions = normed_head_attentions[:,:,:num_classifier_tokens,:num_classifier_tokens]\n",
    "                    logits = normed_head_classifier_model(normed_head_attentions)\n",
    "                    label_ids = torch.argmax(logits, dim=-1)\n",
    "                    labels = [normed_id2label[int(label_id.item())] for label_id in label_ids]\n",
    "                    if len(normed_attention_head_types[layer]) == 0:\n",
    "                        for i in range(len(labels)):\n",
    "                            normed_attention_head_types[layer].append(Counter())\n",
    "                    for i, label in enumerate(labels):\n",
    "                        normed_attention_head_types[layer][i][label] += 1\n",
    "                del attentions, allalpha_f\n",
    "            k += 1\n",
    "            if k == 100:\n",
    "                break\n",
    "        for layer in attention_head_types:\n",
    "            for head_type_ctr in layer:\n",
    "                attention_counter += head_type_ctr\n",
    "        total_counter = Counter()\n",
    "        for layer in normed_attention_head_types:\n",
    "            for head_type_ctr in layer:\n",
    "                normed_attention_counter += head_type_ctr\n",
    "    print(task)\n",
    "    print(\"attention:\", {k:v/sum(attention_counter.values()) for k,v in attention_counter.most_common()})\n",
    "    print(\"weight normed:\", {k:v/sum(normed_attention_counter.values()) for k,v in normed_attention_counter.most_common()})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SST-2, STS-B, MNLI, RTE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Finetuned Model - All"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CoLA\n",
      "attention: {'mix': 0.43254166666666666, 'other': 0.42105555555555557, 'diagonal': 0.07380555555555555, 'vertical': 0.06659722222222222, 'block': 0.006}\n",
      "weight normed: {'Heterogeneous': 0.4975833333333333, 'Diagonal': 0.26311111111111113, 'Vertical + Diagonal': 0.10448611111111111, 'Block': 0.08558333333333333, 'Vertical': 0.04923611111111111}\n",
      "SST-2\n",
      "attention: {'mix': 0.41273611111111114, 'other': 0.2281111111111111, 'diagonal': 0.21891666666666668, 'vertical': 0.08775, 'block': 0.05248611111111111}\n",
      "weight normed: {'Diagonal': 0.46276388888888886, 'Heterogeneous': 0.31830555555555556, 'Block': 0.11891666666666667, 'Vertical + Diagonal': 0.07868055555555556, 'Vertical': 0.021333333333333333}\n",
      "MRPC\n",
      "attention: {'mix': 0.35806944444444444, 'other': 0.23297222222222222, 'diagonal': 0.19951388888888888, 'vertical': 0.14416666666666667, 'block': 0.06527777777777778}\n",
      "weight normed: {'Diagonal': 0.5124722222222222, 'Heterogeneous': 0.3212083333333333, 'Vertical + Diagonal': 0.09981944444444445, 'Block': 0.044055555555555556, 'Vertical': 0.022444444444444444}\n",
      "STS-B\n",
      "attention: {'other': 0.49833333333333335, 'mix': 0.24806944444444445, 'diagonal': 0.15319444444444444, 'vertical': 0.07098611111111111, 'block': 0.029416666666666667}\n",
      "weight normed: {'Diagonal': 0.40873611111111113, 'Heterogeneous': 0.358375, 'Vertical + Diagonal': 0.09034722222222222, 'Block': 0.08125, 'Vertical': 0.06129166666666667}\n",
      "QQP\n",
      "attention: {'other': 0.5084166666666666, 'mix': 0.27345833333333336, 'diagonal': 0.16570833333333335, 'block': 0.028, 'vertical': 0.024416666666666666}\n",
      "weight normed: {'Diagonal': 0.42756944444444445, 'Heterogeneous': 0.3610138888888889, 'Block': 0.11218055555555556, 'Vertical + Diagonal': 0.06244444444444444, 'Vertical': 0.03679166666666667}\n",
      "MNLI\n",
      "attention: {'mix': 0.41256944444444443, 'other': 0.25230555555555556, 'diagonal': 0.20425, 'vertical': 0.07290277777777777, 'block': 0.057972222222222224}\n",
      "weight normed: {'Diagonal': 0.5145972222222223, 'Heterogeneous': 0.2991388888888889, 'Block': 0.09234722222222222, 'Vertical + Diagonal': 0.07704166666666666, 'Vertical': 0.016875}\n",
      "QNLI\n",
      "attention: {'other': 0.49675, 'mix': 0.20252777777777778, 'diagonal': 0.197, 'block': 0.06926388888888889, 'vertical': 0.034458333333333334}\n",
      "weight normed: {'Diagonal': 0.5562777777777778, 'Heterogeneous': 0.28179166666666666, 'Vertical + Diagonal': 0.08655555555555555, 'Block': 0.069625, 'Vertical': 0.00575}\n",
      "RTE\n",
      "attention: {'mix': 0.39981944444444445, 'diagonal': 0.2066388888888889, 'other': 0.15740277777777778, 'vertical': 0.12484722222222222, 'block': 0.11129166666666666}\n",
      "weight normed: {'Diagonal': 0.5307222222222222, 'Heterogeneous': 0.3015, 'Vertical + Diagonal': 0.11202777777777778, 'Block': 0.04777777777777778, 'Vertical': 0.007972222222222223}\n"
     ]
    }
   ],
   "source": [
    "set_seed(1337)\n",
    "for task in [\"CoLA\", \"SST-2\", \"MRPC\", \"STS-B\", \"QQP\", \"MNLI\", \"QNLI\", \"RTE\"]:\n",
    "    attention_counter = Counter()\n",
    "    normed_attention_counter = Counter()\n",
    "    for seed in [\"seed_1337\", \"seed_42\", \"seed_86\", \"seed_71\", \"seed_166\"]:\n",
    "        #Load Model\n",
    "        model_path = f\"../models/finetuned/{task}/{seed}/\"\n",
    "        tokenizer = BertTokenizer.from_pretrained(model_path)\n",
    "        config = BertConfig.from_pretrained(model_path)\n",
    "        config.output_attentions = True\n",
    "        transformer_model = BertForSequenceClassification.from_pretrained(model_path,  config=config)\n",
    "        transformer_model = transformer_model.eval()\n",
    "        transformer_model.cuda()\n",
    "        args = Namespace(data_dir=f\"../data/glue/{task}/\", local_rank=-1, \n",
    "                         model_name_or_path=model_path, \n",
    "                         overwrite_cache=False, model_type=\"bert\", max_seq_length=128)\n",
    "        eval_dataset = load_and_cache_examples(args, task.lower(), tokenizer, evaluate=True)\n",
    "        eval_sampler = RandomSampler(eval_dataset)\n",
    "        eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=1)\n",
    "        input_data = None\n",
    "        attention_head_types = [[] for _ in range(12)]\n",
    "        normed_attention_head_types = [[] for _ in range(12)]\n",
    "\n",
    "        k = 0\n",
    "        for batch in eval_dataloader:\n",
    "            batch = tuple(t.to(\"cuda:0\") for t in batch)\n",
    "            n_tokens = batch[1].sum()\n",
    "            if n_tokens < min_max_size[0]:\n",
    "                continue\n",
    "            input_data =  {\"input_ids\": batch[0], \"attention_mask\": batch[1], \"labels\": batch[3]}\n",
    "            with torch.no_grad():\n",
    "                _, _, attentions, allalpha_f, attention_mask = transformer_model(**input_data)\n",
    "                for layer in range(len(attentions)):\n",
    "                    if attentions[layer] is None:\n",
    "                        continue\n",
    "\n",
    "                    num_classifier_tokens = max(n_tokens, min_max_size[0]+20)\n",
    "                    head_attentions = attentions[layer].transpose(0, 1)\n",
    "\n",
    "                    head_attentions = head_attentions[:,:,:num_classifier_tokens,:num_classifier_tokens]\n",
    "                    logits = head_classifier_model(head_attentions)\n",
    "                    label_ids = torch.argmax(logits, dim=-1)\n",
    "                    labels = [id2label[int(label_id.item())] for label_id in label_ids]\n",
    "                    if len(attention_head_types[layer]) == 0:\n",
    "                        for i in range(len(labels)):\n",
    "                            attention_head_types[layer].append(Counter())\n",
    "                    for i, label in enumerate(labels):\n",
    "                        attention_head_types[layer][i][label] += 1\n",
    "                    \n",
    "                    num_classifier_tokens = max(n_tokens, normed_min_max_size[0]+20)\n",
    "                    normed_alpha_f = allalpha_f[layer].norm(dim=-1)\n",
    "                    normed_head_attentions = normed_alpha_f.transpose(0, 1)\n",
    "\n",
    "                    normed_head_attentions = normed_head_attentions[:,:,:num_classifier_tokens,:num_classifier_tokens]\n",
    "                    logits = normed_head_classifier_model(normed_head_attentions)\n",
    "                    label_ids = torch.argmax(logits, dim=-1)\n",
    "                    labels = [normed_id2label[int(label_id.item())] for label_id in label_ids]\n",
    "                    if len(normed_attention_head_types[layer]) == 0:\n",
    "                        for i in range(len(labels)):\n",
    "                            normed_attention_head_types[layer].append(Counter())\n",
    "                    for i, label in enumerate(labels):\n",
    "                        normed_attention_head_types[layer][i][label] += 1\n",
    "                del attentions, allalpha_f\n",
    "            k += 1\n",
    "            if k == 100:\n",
    "                break\n",
    "        for layer in attention_head_types:\n",
    "            for head_type_ctr in layer:\n",
    "                attention_counter += head_type_ctr\n",
    "        total_counter = Counter()\n",
    "        for layer in normed_attention_head_types:\n",
    "            for head_type_ctr in layer:\n",
    "                normed_attention_counter += head_type_ctr\n",
    "    print(task)\n",
    "    print(\"attention:\", {k:v/sum(attention_counter.values()) for k,v in attention_counter.most_common()})\n",
    "    print(\"weight normed:\", {k:v/sum(normed_attention_counter.values()) for k,v in normed_attention_counter.most_common()})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pre-trained Super Survivors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CoLA\n",
      "attention: {'mix': 0.4440588235294118, 'other': 0.3938235294117647, 'diagonal': 0.13805882352941176, 'vertical': 0.01764705882352941, 'block': 0.006411764705882353}\n",
      "weight normed: {'Diagonal': 0.42164705882352943, 'Heterogeneous': 0.34647058823529414, 'Vertical + Diagonal': 0.10358823529411765, 'Block': 0.09470588235294118, 'Vertical': 0.03358823529411765}\n",
      "SST-2\n",
      "attention: {'other': 0.35326315789473683, 'diagonal': 0.2556842105263158, 'mix': 0.19936842105263158, 'block': 0.18852631578947368, 'vertical': 0.003157894736842105}\n",
      "weight normed: {'Block': 0.3336842105263158, 'Heterogeneous': 0.2963157894736842, 'Vertical + Diagonal': 0.18610526315789475, 'Diagonal': 0.16736842105263158, 'Vertical': 0.016526315789473684}\n",
      "MRPC\n",
      "attention: {'vertical': 0.38671428571428573, 'block': 0.2302142857142857, 'other': 0.19514285714285715, 'diagonal': 0.09864285714285714, 'mix': 0.08928571428571429}\n",
      "weight normed: {'Vertical': 0.2790714285714286, 'Heterogeneous': 0.26971428571428574, 'Vertical + Diagonal': 0.23442857142857143, 'Diagonal': 0.17257142857142857, 'Block': 0.04421428571428571}\n",
      "STS-B\n",
      "attention: {'other': 0.6685833333333333, 'block': 0.1305, 'diagonal': 0.1125, 'mix': 0.06608333333333333, 'vertical': 0.022333333333333334}\n",
      "weight normed: {'Heterogeneous': 0.4435, 'Block': 0.26958333333333334, 'Diagonal': 0.13016666666666668, 'Vertical + Diagonal': 0.12333333333333334, 'Vertical': 0.033416666666666664}\n",
      "QQP\n",
      "attention: {'other': 0.558695652173913, 'vertical': 0.14869565217391303, 'diagonal': 0.1274782608695652, 'mix': 0.11991304347826087, 'block': 0.04521739130434783}\n",
      "weight normed: {'Vertical': 0.416695652173913, 'Vertical + Diagonal': 0.19165217391304348, 'Heterogeneous': 0.16973913043478261, 'Diagonal': 0.15539130434782608, 'Block': 0.06652173913043478}\n",
      "MNLI\n",
      "attention: {'other': 0.36845, 'diagonal': 0.22575, 'block': 0.2244, 'mix': 0.12245, 'vertical': 0.05895}\n",
      "weight normed: {'Block': 0.29645, 'Diagonal': 0.25115, 'Heterogeneous': 0.24015, 'Vertical + Diagonal': 0.1935, 'Vertical': 0.01875}\n",
      "QNLI\n",
      "attention: {'other': 0.5337419354838709, 'diagonal': 0.2238709677419355, 'block': 0.16096774193548388, 'mix': 0.050451612903225806, 'vertical': 0.03096774193548387}\n",
      "weight normed: {'Diagonal': 0.33948387096774196, 'Heterogeneous': 0.2691612903225806, 'Vertical + Diagonal': 0.19161290322580646, 'Vertical': 0.10619354838709677, 'Block': 0.0935483870967742}\n",
      "RTE\n",
      "attention: {'block': 0.5581818181818182, 'other': 0.15763636363636363, 'diagonal': 0.15763636363636363, 'mix': 0.09927272727272728, 'vertical': 0.02727272727272727}\n",
      "weight normed: {'Heterogeneous': 0.46654545454545454, 'Vertical + Diagonal': 0.2270909090909091, 'Block': 0.16436363636363635, 'Diagonal': 0.1398181818181818, 'Vertical': 0.002181818181818182}\n"
     ]
    }
   ],
   "source": [
    "set_seed(1337)\n",
    "for task in [\"CoLA\", \"SST-2\", \"MRPC\", \"STS-B\", \"QQP\", \"MNLI\", \"QNLI\", \"RTE\"]:\n",
    "    attention_counter = Counter()\n",
    "    normed_attention_counter = Counter()\n",
    "    for seed in [\"seed_1337\", \"seed_42\", \"seed_86\", \"seed_71\", \"seed_166\"]:\n",
    "        #Load Model\n",
    "        model_path = f\"../models/finetuned/{task}/{seed}/\"\n",
    "        tokenizer = BertTokenizer.from_pretrained(model_path)\n",
    "        config = BertConfig.from_pretrained(model_path)\n",
    "        config.output_attentions = True\n",
    "        transformer_model = BertForSequenceClassification.from_pretrained('bert-base-uncased',  config=config)\n",
    "        # Prune\n",
    "        mask_path = f\"../masks/heads_mlps_super/{task}/{seed}/\"\n",
    "        head_mask = np.load(f\"{mask_path}/head_mask.npy\")\n",
    "        mlp_mask = np.load(f\"{mask_path}/mlp_mask.npy\")\n",
    "        head_mask = torch.from_numpy(head_mask)\n",
    "        heads_to_prune = {} \n",
    "        for layer in range(len(head_mask)):\n",
    "            heads_to_mask = [h[0] for h in (1 - head_mask[layer].long()).nonzero().tolist()]\n",
    "            heads_to_prune[layer] = heads_to_mask\n",
    "        mlps_to_prune = [h[0] for h in (1 - torch.from_numpy(mlp_mask).long()).nonzero().tolist()]\n",
    "\n",
    "        transformer_model.prune_heads(heads_to_prune)\n",
    "        transformer_model.prune_mlps(mlps_to_prune)\n",
    "        transformer_model = transformer_model.eval()\n",
    "        transformer_model.cuda()\n",
    "        args = Namespace(data_dir=f\"../data/glue/{task}/\", local_rank=-1, \n",
    "                         model_name_or_path=model_path, \n",
    "                         overwrite_cache=False, model_type=\"bert\", max_seq_length=128)\n",
    "        eval_dataset = load_and_cache_examples(args, task.lower(), tokenizer, evaluate=True)\n",
    "        eval_sampler = RandomSampler(eval_dataset)\n",
    "        eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=1)\n",
    "        input_data = None\n",
    "        attention_head_types = [[] for _ in range(12)]\n",
    "        normed_attention_head_types = [[] for _ in range(12)]\n",
    "\n",
    "        k = 0\n",
    "        for batch in eval_dataloader:\n",
    "            batch = tuple(t.to(\"cuda:0\") for t in batch)\n",
    "            n_tokens = batch[1].sum()\n",
    "            if n_tokens < min_max_size[0]:\n",
    "                continue\n",
    "            input_data =  {\"input_ids\": batch[0], \"attention_mask\": batch[1], \"labels\": batch[3]}\n",
    "            with torch.no_grad():\n",
    "                _, _, attentions, allalpha_f, attention_mask = transformer_model(**input_data)\n",
    "                for layer in range(len(attentions)):\n",
    "                    if attentions[layer] is None:\n",
    "                        continue\n",
    "\n",
    "                    num_classifier_tokens = max(n_tokens, min_max_size[0]+20)\n",
    "                    head_attentions = attentions[layer].transpose(0, 1)\n",
    "\n",
    "                    head_attentions = head_attentions[:,:,:num_classifier_tokens,:num_classifier_tokens]\n",
    "                    logits = head_classifier_model(head_attentions)\n",
    "                    label_ids = torch.argmax(logits, dim=-1)\n",
    "                    labels = [id2label[int(label_id.item())] for label_id in label_ids]\n",
    "                    if len(attention_head_types[layer]) == 0:\n",
    "                        for i in range(len(labels)):\n",
    "                            attention_head_types[layer].append(Counter())\n",
    "                    for i, label in enumerate(labels):\n",
    "                        attention_head_types[layer][i][label] += 1\n",
    "                    \n",
    "                    num_classifier_tokens = max(n_tokens, normed_min_max_size[0]+20)\n",
    "                    normed_alpha_f = allalpha_f[layer].norm(dim=-1)\n",
    "                    normed_head_attentions = normed_alpha_f.transpose(0, 1)\n",
    "\n",
    "                    normed_head_attentions = normed_head_attentions[:,:,:num_classifier_tokens,:num_classifier_tokens]\n",
    "                    logits = normed_head_classifier_model(normed_head_attentions)\n",
    "                    label_ids = torch.argmax(logits, dim=-1)\n",
    "                    labels = [normed_id2label[int(label_id.item())] for label_id in label_ids]\n",
    "                    if len(normed_attention_head_types[layer]) == 0:\n",
    "                        for i in range(len(labels)):\n",
    "                            normed_attention_head_types[layer].append(Counter())\n",
    "                    for i, label in enumerate(labels):\n",
    "                        normed_attention_head_types[layer][i][label] += 1\n",
    "                del attentions, allalpha_f\n",
    "            k += 1\n",
    "            if k == 100:\n",
    "                break\n",
    "        for layer in attention_head_types:\n",
    "            for head_type_ctr in layer:\n",
    "                attention_counter += head_type_ctr\n",
    "        total_counter = Counter()\n",
    "        for layer in normed_attention_head_types:\n",
    "            for head_type_ctr in layer:\n",
    "                normed_attention_counter += head_type_ctr\n",
    "    print(task)\n",
    "    print(\"attention:\", {k:v/sum(attention_counter.values()) for k,v in attention_counter.most_common()})\n",
    "    print(\"weight normed:\", {k:v/sum(normed_attention_counter.values()) for k,v in normed_attention_counter.most_common()})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pre-trained All"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CoLA\n",
      "attention: {'mix': 0.43969444444444444, 'other': 0.41041666666666665, 'vertical': 0.07502777777777778, 'diagonal': 0.06875, 'block': 0.006111111111111111}\n",
      "weight normed: {'Heterogeneous': 0.5134583333333333, 'Diagonal': 0.23484722222222223, 'Vertical + Diagonal': 0.11059722222222222, 'Block': 0.07826388888888888, 'Vertical': 0.06283333333333334}\n",
      "SST-2\n",
      "attention: {'mix': 0.40734722222222225, 'other': 0.225, 'diagonal': 0.20219444444444445, 'vertical': 0.11197222222222222, 'block': 0.05348611111111111}\n",
      "weight normed: {'Diagonal': 0.424625, 'Heterogeneous': 0.33469444444444446, 'Block': 0.10854166666666666, 'Vertical + Diagonal': 0.0976111111111111, 'Vertical': 0.034527777777777775}\n",
      "MRPC\n",
      "attention: {'mix': 0.358625, 'other': 0.2555, 'vertical': 0.16709722222222223, 'diagonal': 0.16061111111111112, 'block': 0.058166666666666665}\n",
      "weight normed: {'Diagonal': 0.4820833333333333, 'Heterogeneous': 0.32166666666666666, 'Vertical + Diagonal': 0.13972222222222222, 'Block': 0.041777777777777775, 'Vertical': 0.01475}\n",
      "STS-B\n",
      "attention: {'other': 0.5340138888888889, 'mix': 0.2610972222222222, 'diagonal': 0.1284861111111111, 'vertical': 0.05309722222222222, 'block': 0.023305555555555555}\n",
      "weight normed: {'Heterogeneous': 0.40381944444444445, 'Diagonal': 0.3857083333333333, 'Vertical + Diagonal': 0.11375, 'Block': 0.07983333333333334, 'Vertical': 0.016888888888888887}\n",
      "QQP\n",
      "attention: {'other': 0.5591805555555556, 'mix': 0.23593055555555556, 'diagonal': 0.14372222222222222, 'vertical': 0.041166666666666664, 'block': 0.02}\n",
      "weight normed: {'Diagonal': 0.4167222222222222, 'Heterogeneous': 0.370875, 'Vertical + Diagonal': 0.09765277777777778, 'Block': 0.09270833333333334, 'Vertical': 0.022041666666666668}\n",
      "MNLI\n",
      "attention: {'mix': 0.385125, 'other': 0.282625, 'diagonal': 0.17231944444444444, 'vertical': 0.10627777777777778, 'block': 0.05365277777777778}\n",
      "weight normed: {'Diagonal': 0.44180555555555556, 'Heterogeneous': 0.3438333333333333, 'Vertical + Diagonal': 0.11344444444444444, 'Block': 0.08697222222222223, 'Vertical': 0.013944444444444445}\n",
      "QNLI\n",
      "attention: {'other': 0.5695833333333333, 'diagonal': 0.17322222222222222, 'mix': 0.1458472222222222, 'block': 0.05686111111111111, 'vertical': 0.05448611111111111}\n",
      "weight normed: {'Diagonal': 0.475, 'Heterogeneous': 0.3186388888888889, 'Vertical + Diagonal': 0.12791666666666668, 'Block': 0.06390277777777778, 'Vertical': 0.014541666666666666}\n",
      "RTE\n",
      "attention: {'mix': 0.3686527777777778, 'diagonal': 0.19225, 'vertical': 0.16768055555555555, 'other': 0.158625, 'block': 0.11279166666666667}\n",
      "weight normed: {'Diagonal': 0.4802638888888889, 'Heterogeneous': 0.32993055555555556, 'Vertical + Diagonal': 0.13279166666666667, 'Block': 0.04620833333333333, 'Vertical': 0.010805555555555556}\n"
     ]
    }
   ],
   "source": [
    "set_seed(1337)\n",
    "for task in [\"CoLA\", \"SST-2\", \"MRPC\", \"STS-B\", \"QQP\", \"MNLI\", \"QNLI\", \"RTE\"]:\n",
    "    attention_counter = Counter()\n",
    "    normed_attention_counter = Counter()\n",
    "    for seed in [\"seed_1337\", \"seed_42\", \"seed_86\", \"seed_71\", \"seed_166\"]:\n",
    "        #Load Model\n",
    "        model_path = f\"../models/finetuned/{task}/{seed}/\"\n",
    "        tokenizer = BertTokenizer.from_pretrained(model_path)\n",
    "        config = BertConfig.from_pretrained(model_path)\n",
    "        config.output_attentions = True\n",
    "        transformer_model = BertForSequenceClassification.from_pretrained('bert-base-uncased',  config=config)\n",
    "        transformer_model = transformer_model.eval()\n",
    "        \n",
    "        transformer_model.cuda()\n",
    "        args = Namespace(data_dir=f\"../data/glue/{task}/\", local_rank=-1, \n",
    "                         model_name_or_path=model_path, \n",
    "                         overwrite_cache=False, model_type=\"bert\", max_seq_length=128)\n",
    "        eval_dataset = load_and_cache_examples(args, task.lower(), tokenizer, evaluate=True)\n",
    "        eval_sampler = RandomSampler(eval_dataset)\n",
    "        eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=1)\n",
    "        input_data = None\n",
    "        attention_head_types = [[] for _ in range(12)]\n",
    "        normed_attention_head_types = [[] for _ in range(12)]\n",
    "\n",
    "        k = 0\n",
    "        for batch in eval_dataloader:\n",
    "            batch = tuple(t.to(\"cuda:0\") for t in batch)\n",
    "            n_tokens = batch[1].sum()\n",
    "            if n_tokens < min_max_size[0]:\n",
    "                continue\n",
    "            input_data =  {\"input_ids\": batch[0], \"attention_mask\": batch[1], \"labels\": batch[3]}\n",
    "            with torch.no_grad():\n",
    "                _, _, attentions, allalpha_f, attention_mask = transformer_model(**input_data)\n",
    "                for layer in range(len(attentions)):\n",
    "                    if attentions[layer] is None:\n",
    "                        continue\n",
    "\n",
    "                    num_classifier_tokens = max(n_tokens, min_max_size[0]+20)\n",
    "                    head_attentions = attentions[layer].transpose(0, 1)\n",
    "\n",
    "                    head_attentions = head_attentions[:,:,:num_classifier_tokens,:num_classifier_tokens]\n",
    "                    logits = head_classifier_model(head_attentions)\n",
    "                    label_ids = torch.argmax(logits, dim=-1)\n",
    "                    labels = [id2label[int(label_id.item())] for label_id in label_ids]\n",
    "                    if len(attention_head_types[layer]) == 0:\n",
    "                        for i in range(len(labels)):\n",
    "                            attention_head_types[layer].append(Counter())\n",
    "                    for i, label in enumerate(labels):\n",
    "                        attention_head_types[layer][i][label] += 1\n",
    "                    \n",
    "                    num_classifier_tokens = max(n_tokens, normed_min_max_size[0]+20)\n",
    "                    normed_alpha_f = allalpha_f[layer].norm(dim=-1)\n",
    "                    normed_head_attentions = normed_alpha_f.transpose(0, 1)\n",
    "\n",
    "                    normed_head_attentions = normed_head_attentions[:,:,:num_classifier_tokens,:num_classifier_tokens]\n",
    "                    logits = normed_head_classifier_model(normed_head_attentions)\n",
    "                    label_ids = torch.argmax(logits, dim=-1)\n",
    "                    labels = [normed_id2label[int(label_id.item())] for label_id in label_ids]\n",
    "                    if len(normed_attention_head_types[layer]) == 0:\n",
    "                        for i in range(len(labels)):\n",
    "                            normed_attention_head_types[layer].append(Counter())\n",
    "                    for i, label in enumerate(labels):\n",
    "                        normed_attention_head_types[layer][i][label] += 1\n",
    "                del attentions, allalpha_f\n",
    "            k += 1\n",
    "            if k == 100:\n",
    "                break\n",
    "        for layer in attention_head_types:\n",
    "            for head_type_ctr in layer:\n",
    "                attention_counter += head_type_ctr\n",
    "        total_counter = Counter()\n",
    "        for layer in normed_attention_head_types:\n",
    "            for head_type_ctr in layer:\n",
    "                normed_attention_counter += head_type_ctr\n",
    "    print(task)\n",
    "    print(\"attention:\", {k:v/sum(attention_counter.values()) for k,v in attention_counter.most_common()})\n",
    "    print(\"weight normed:\", {k:v/sum(normed_attention_counter.values()) for k,v in normed_attention_counter.most_common()})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Correlation coefficient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "heterogeneous = []\n",
    "survivors = []\n",
    "\n",
    "set_seed(1337)\n",
    "for task in [\"CoLA\", \"SST-2\", \"MRPC\", \"STS-B\", \"QQP\", \"MNLI\", \"QNLI\", \"RTE\"]:\n",
    "    attention_counter = Counter()\n",
    "    normed_attention_counter = Counter()\n",
    "    for seed in [\"seed_1337\", \"seed_42\", \"seed_86\", \"seed_71\", \"seed_166\"]:\n",
    "        #Load Model\n",
    "        model_path = f\"../models/finetuned/{task}/{seed}/\"\n",
    "        tokenizer = BertTokenizer.from_pretrained(model_path)\n",
    "        config = BertConfig.from_pretrained(model_path)\n",
    "        config.output_attentions = True\n",
    "        transformer_model = BertForSequenceClassification.from_pretrained(model_path,  config=config)\n",
    "        mask_path = f\"../masks/heads_mlps_super/{task}/{seed}/\"\n",
    "        head_mask = np.load(f\"{mask_path}/head_mask.npy\")\n",
    "        mlp_mask = np.load(f\"{mask_path}/mlp_mask.npy\")\n",
    "        head_mask = torch.from_numpy(head_mask)\n",
    "        heads_to_prune = {} \n",
    "        for layer in range(len(head_mask)):\n",
    "            heads_to_mask = [h[0] for h in (1 - head_mask[layer].long()).nonzero().tolist()]\n",
    "            heads_to_prune[layer] = heads_to_mask\n",
    "        mlps_to_prune = [h[0] for h in (1 - torch.from_numpy(mlp_mask).long()).nonzero().tolist()]\n",
    "        \n",
    "        \n",
    "        transformer_model = transformer_model.eval()\n",
    "        transformer_model.cuda()\n",
    "        args = Namespace(data_dir=f\"../data/glue/{task}/\", local_rank=-1, \n",
    "                         model_name_or_path=model_path, \n",
    "                         overwrite_cache=False, model_type=\"bert\", max_seq_length=128)\n",
    "        eval_dataset = load_and_cache_examples(args, task.lower(), tokenizer, evaluate=True)\n",
    "        eval_sampler = RandomSampler(eval_dataset)\n",
    "        eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=1)\n",
    "        input_data = None\n",
    "        attention_head_types = [[] for _ in range(12)]\n",
    "        normed_attention_head_types = [[] for _ in range(12)]\n",
    "\n",
    "        k = 0\n",
    "        for batch in eval_dataloader:\n",
    "            batch = tuple(t.to(\"cuda:0\") for t in batch)\n",
    "            n_tokens = batch[1].sum()\n",
    "            if n_tokens < min_max_size[0]:\n",
    "                continue\n",
    "            input_data =  {\"input_ids\": batch[0], \"attention_mask\": batch[1], \"labels\": batch[3]}\n",
    "            with torch.no_grad():\n",
    "                _, _, attentions, allalpha_f, attention_mask = transformer_model(**input_data)\n",
    "                for layer in range(len(attentions)):\n",
    "                    if attentions[layer] is None:\n",
    "                        continue\n",
    "                    \n",
    "                    num_classifier_tokens = max(n_tokens, normed_min_max_size[0]+20)\n",
    "                    normed_alpha_f = allalpha_f[layer].norm(dim=-1)\n",
    "                    normed_head_attentions = normed_alpha_f.transpose(0, 1)\n",
    "\n",
    "                    normed_head_attentions = normed_head_attentions[:,:,:num_classifier_tokens,:num_classifier_tokens]\n",
    "                    logits = normed_head_classifier_model(normed_head_attentions)\n",
    "                    label_ids = torch.argmax(logits, dim=-1)\n",
    "                    labels = [normed_id2label[int(label_id.item())] for label_id in label_ids]\n",
    "                    \n",
    "                    for head, label in enumerate(labels):\n",
    "                        \n",
    "                        is_heterogeneous = label == 'Heterogeneous'\n",
    "                        is_survivor = head not in heads_to_prune[layer]\n",
    "                        if is_heterogeneous:\n",
    "                            heterogeneous.append(1.)\n",
    "                        else:\n",
    "                            heterogeneous.append(0.)\n",
    "                        if is_survivor:\n",
    "                            survivors.append(1.)\n",
    "                        else:\n",
    "                            survivors.append(0.)\n",
    "                del attentions, allalpha_f\n",
    "            k += 1\n",
    "            if k == 100:\n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import pearsonr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.0022907000419617545, 0.08211965091231407)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pearsonr(heterogeneous, survivors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Attention Patterns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "patterns = {\n",
    "    \"CoLA\": {\n",
    "      \"attention\": {'mix': 0.4218823529411765, 'other': 0.4046470588235294, 'diagonal': 0.1458235294117647, 'vertical': 0.02123529411764706, 'block': 0.006411764705882353},\n",
    "      \"weight normed\": {'Diagonal': 0.42823529411764705, 'Heterogeneous': 0.3382941176470588, 'Vertical + Diagonal': 0.10252941176470588, 'Block': 0.09682352941176471, 'Vertical': 0.03411764705882353}\n",
    "    },\n",
    "    \"SST-2\": {\n",
    "      \"attention\": {'other': 0.3487368421052632, 'diagonal': 0.25705263157894737, 'block': 0.1965263157894737, 'mix': 0.19473684210526315, 'vertical': 0.0029473684210526317},\n",
    "      \"weight normed\": {'Block': 0.36326315789473684, 'Heterogeneous': 0.26063157894736844, 'Diagonal': 0.18894736842105264, 'Vertical + Diagonal': 0.17410526315789474, 'Vertical': 0.013052631578947368},\n",
    "    },\n",
    "    \"MRPC\": {\n",
    "      \"attention\": {'vertical': 0.377, 'block': 0.23685714285714285, 'other': 0.18492857142857144, 'diagonal': 0.10107142857142858, 'mix': 0.10014285714285714},\n",
    "      \"weight normed\": {'Heterogeneous': 0.2807857142857143, 'Vertical': 0.25007142857142856, 'Vertical + Diagonal': 0.23157142857142857, 'Diagonal': 0.18985714285714286, 'Block': 0.047714285714285716}\n",
    "    },\n",
    "    \"STS-B\": {\n",
    "      \"attention\": {'other': 0.6768333333333333, 'block': 0.13475, 'diagonal': 0.1115, 'mix': 0.059583333333333335, 'vertical': 0.017333333333333333},\n",
    "      \"weight normed\": {'Heterogeneous': 0.438, 'Block': 0.28391666666666665, 'Diagonal': 0.1285, 'Vertical + Diagonal': 0.12166666666666667, 'Vertical': 0.027916666666666666}\n",
    "    },\n",
    "    \"QQP\": {\n",
    "      \"attention\": {'other': 0.5302608695652173, 'mix': 0.14478260869565218, 'diagonal': 0.1376521739130435, 'vertical': 0.13460869565217393, 'block': 0.052695652173913046},\n",
    "      \"weight normed\":  {'Vertical': 0.34956521739130436, 'Vertical + Diagonal': 0.1988695652173913, 'Heterogeneous': 0.198, 'Diagonal': 0.16643478260869565, 'Block': 0.08713043478260869}\n",
    "    },\n",
    "    \"MNLI\": {\n",
    "      \"attention\": {'other': 0.354, 'diagonal': 0.2806, 'block': 0.2482, 'mix': 0.0948, 'vertical': 0.0224},\n",
    "      \"weight normed\": {'Diagonal': 0.3262, 'Block': 0.2945, 'Heterogeneous': 0.2137, 'Vertical + Diagonal': 0.1589, 'Vertical': 0.0067}\n",
    "    },\n",
    "    \"QNLI\": {\n",
    "      \"attention\": {'other': 0.5126451612903226, 'diagonal': 0.2372258064516129, 'block': 0.16548387096774195, 'mix': 0.05987096774193548, 'vertical': 0.024774193548387096},\n",
    "      \"weight normed\": {'Diagonal': 0.36316129032258065, 'Heterogeneous': 0.2718709677419355, 'Vertical + Diagonal': 0.16612903225806452, 'Block': 0.10541935483870968, 'Vertical': 0.09341935483870968}\n",
    "    },\n",
    "    \"RTE\": {\n",
    "      \"attention\": {'block': 0.5658181818181818, 'diagonal': 0.158, 'other': 0.15454545454545454, 'mix': 0.09945454545454545, 'vertical': 0.02218181818181818},\n",
    "      \"weight normed\": {'Heterogeneous': 0.46036363636363636, 'Vertical + Diagonal': 0.22927272727272727, 'Block': 0.1698181818181818, 'Diagonal': 0.1390909090909091, 'Vertical': 0.0014545454545454545}  \n",
    "    },\n",
    "\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAAD7CAYAAADJukfwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAUv0lEQVR4nO3dfXBU9aHG8Sdvi7wEyKAZO2KLqAGhldZbsQ1MDSgINNcYGV601BZBIAwITLCaoiiOikBDx4iAtSPYdIogCcRrYHAYBzoUiTNgX0QuUcAiUCICIeSmJJvN7/5BsyMkmyUnZ3/s2f1+ZviD3T1nn5yTfXLO77xsgjHGCABgReLVDgAA8YTSBQCLKF0AsIjSBQCLKF0AsIjSBQCLKF0AsCg53AvOnv0/NTW5fypvr17ddPp0revzdRMZ3eOFnF7IKHkjpxcySpHJmZiYoLS0riGfD1u6TU0mIqXbPO9oR0b3eCGnFzJK3sjphYyS/ZwMLwCARZQuAFhE6QKARZQuAFhE6QKARZQuAFhE6QKARQnhbmJ++nSto/PYuvfsok4pSY5C1fsDqqmuczRteznN6YWMkjdyeiGjZC+nFzJKrO9QEhMT1KtXt5DPh704wqlOKUl6bMs+R9O+MeYOl9OE5jSnFzJK3sjphYySvZxeyCixvp1ieAEALKJ0AcAiShcALKJ0AcAiShcALKJ0AcAiShcALKJ0AcAiShcALIrYFWkAYltDIODoqq2GQCACabyD0gXgiC8pSXvff6Ld0/3XyGURSOMdlC6AmOV0a7x52kigdAE4Egj4HW21BgL+CKRpndOtcSlyW+SULgBHkpJSVFhY2O7p8vPzJV1wP5BHULoAYpbTrfHmaSMhYqXbEGjqwFhKk8tp2novjsACscrp1rgUuS3yiJWuLymxgz+sHRyBRbSJxoM/cA/DC0CUicaDP3APpQtEmWgch4R7KF3EFS/sukfjOCTcQ+kirrDrjquNG94AgEVs6XoAp98BsYPS9QBOvwNiB8MLAGBR3G/peuGmHQBiR9yXLjftAGBT3JcugNjV6G90fFyj0d/ocpqLKF0AMSs5JVmf/WaXo2lvnT/U5TQXcSANACyidAHAIkoXACyidAHAIkoXACyidAHAIkoXACyidAHAIi6O8IBovKoGkcP6jm2UrgdE41U1iBzWd2xjeAEALKJ0AcAiShcALKJ0AcAiShcALKJ0AcAiShcALKJ0AcAiShcALOKKNLiGr7MHwotY6XL9ePzh6+yB8CJWulw/DgAtMaYLABZRugBgEaULABZx9oIHNPkDjse5m/wBl9MA6AhK1wMSU5K0omCyo2lnLV7jchpvc3paW/O0QEdRuogrTk9rkzi1zYuicS+R0gXgiNNz8W2ehx+Ne4mULgBHnJ6LH+/n4XP2AgBYROkCgEWULgBYROkCgEVxfyDNC0dgAcSOuC9djsACsInhBQCwiNIFAIvifngBgDNOL7GN95swUbpwDQcl44vTS2zj/SZMlC5cw0FJIDzGdAHAIkoXACxieAFxxem4c/O0QEdRunCNF45mOx13lhh7hjvivnS9UBRewdFsILy4L12KAtEmGr9iBu6J+9IFok00fsUM3MPZCwBgEaULABZRugBgUcTGdDkYAAAtRax0ORiAaMTGAK42zl5AXGFjAFcbY7oAYBGlCwAWUboAYBGlCwAWUboAYBGlCwAWUboAYBGlCwAWUboAYBGlCwAWUboAYBGlCwAWUboAYBGlCwAWUboAYBGlCwAWUboAYBGlCwAWJRhjTFsvGDt2nE6ePNnuGaekJOn44f91FOqGvv3lt/R9VE5zeiGj5I2cXsgo2cvphYwS6zuU66+/XiUl74R8PmLfkWaM0Q19+zue1hanOb2QsXlaW1iW7vBCxub3Yn23X9gt3dOna9XU1P43v+66VD2f/56jUAsLs3Xq1HlH07aX05xeyCh5I6cXMkr2cnoho8T6DiUxMUG9enUL/byjNAAARyhdALCI0gUAiyhdALAoYmcveIW/oVELC7MdTQcgujn9fDdPGwlxX7opvmT9JWdsu6cbUlYSgTQA3OT08y1F7jMe96ULIHYF6hscl2egvsHlNBfFfek6XSmRWiEA3JPUyafx6/McTbthwipJ9e4GUgRLNxrHUlrjdKVEaoW0xivLkvFxILyIlW40jqV4lVeWJePjQHgRK91oHEvxKpYlEDsiVrrROJbiVSxLIHZwcQQAWBT3Zy8gvnjloCRiF6WLuOKVg5KIXQwvAIBFlC4AWMTwAoCY1dDY8J8zeJxNGwmULhBl/P6A84N9lr7w0St8yT4dftHZGH7fBSXy1GXAAJxJSUnq4HnZiGaM6QKARZQuAFhE6QKARZQuAFhE6QKARZQuAFhE6QKARZyn6wHReFWNV3FDeFxtlK4HRONVNV7FDeFxtTG8AAAWUboAYFHcDy84HS9lrBSIfk3+hv8MsTmbNhLivnSdjpcyVgpEv8QUn/47v8zRtP9TmKNIfMYZXgAAiyhdALCI0gUAiyhdALCI0gUAiyhdALAoYqeMcb8AAGgpYqXL/QIAoKW4vzgC8YU9MPf4GxodfVW8v6ExAmm8g9JFXGEPzD0pvmT9Jaf9y9LprTVjBQfSAMAitnQBOOL0hvDxfjN4SheIMl4Zd3Z6Q/h4vxk8pYu4Eo23+rsc486xjdKFa7ywuxmNt/pDfIn70nW65WNrq8dL2N0Ewov70nW65cNWDwAnOGUMACyidAHAIkoXACyidAHAIkoXACyidAHAIkoXACyidAHAori/OALucXqjFm4OjnhC6cI1Tm/Uwk1avKmh0e/wj6w/Amm8g9IF4IgvOaUDl9BfcD+QRzCmCwAWUboAYBGlCwAWUboAYFHEDqR54WtRgGjEZye2Rax0+VoUwBk+O7GN4QUAsIjSBQCLKF0AsIjSBQCLKF0AsIjSBQCLKF0AsIjSBQCLKF0AsIjSBQCLKF0AsIjSBQCLKF0AsIjSBQCLKF0AsIhvA4ZrnN58mxtvI55QunCN05tvc+NtxBOGFwDAIrZ0PYDvzAJiB6XrAXxnFhA7GF4AAIsoXQCwiNIFAIsoXQCwiNIFAIsoXQCwiNIFAIsoXQCwiNIFAIsoXQCwiNIFAIsoXQCwiNIFAIvC3mUsMTHB8czT0zo7nrYj79teTnN6IaPkjZxeyCjZy+mFjBLr28k0CcYY4zQQAKB9GF4AAIsoXQCwiNIFAIsoXQCwiNIFAIsoXQCwiNIFAIsoXQCwiNIFAIvCXgZ8JYYPH66Ghgbt3LlTSUlJkqTS0lIVFBTomWeeUZcuXVRQUKDf/va3GjNmTPD5HTt2qKioSMeOHdPYsWNVUVHRYt79+vXTvn371LVrVzeiqqGhQcuXL9f27duVnJysTp06acaMGRo9erQqKir0yCOPKD8/X9OmTZMkVVRUaMmSJSotLW0zz/Dhw7V69WplZGS4krN5nj6fTz6fT36/X7m5uSovL5ck1dXV6auvvlKfPn0kSVlZWcrMzNS0adOCjzXnXbp0qWt5wq3nl156STfccIP8fr969+6tF198Udddd50k6ec//7lOnDihbt26qb6+Xg899JB+8YtfSJJOnTqlZcuWae/everevbuSkpI0fvx4jR8/vsO521rnklRZWaklS5bo6NGjCgQCuu222/TUU0/pxhtvlCQ99dRT2r17t9LS0nThwgWNGDFC8+fP73CurVu36vXXX5cxRvX19Ro4cKAKCwtDPj5u3Dg1NDTI7/friy++0K233ipJGjBggBYvXhyc7759+7R06VLV1NRIku6++2796le/UkKC80tv/X6/Vq5cqS1btsjn8ykpKUk/+tGPlJ+fr5SUlFanefXVV1VXV6cnn3yy1ed37typadOmacWKFRoxYoTjbK0tr6NHj4ZcVi+99JKKior0/vvvKykpSY2NjRo3bpwmT57c6s/wpz/9Senp6TLGqHPnzlq0aJH69+/vOK+MC4YNG2Zyc3PNjh07go9NmjTJ5ObmmuLiYlNSUmKGDRtmRo4cafx+vzHGmJKSEjN79mxjjDFffvmlGTx4cKvzzsjIMLW1tW7ENMYYU1BQYObMmWMuXLhgjDHm4MGDZujQoWb37t1mz549ZsiQISYzM9OcO3fOGGPMnj17TG5ubtg8w4YNMwcPHnQt5+XzPHjwoBk4cKA5efJkq7lCPeZ2nnDruXmdNjU1mblz55qFCxde8toPPvjAGGPMiRMnzB133GEOHDhg6urqzMiRI83q1atNIBAwxhhTU1Nj1q1b50ruttZ5dXW1yczMNFu2bAm+fs2aNWbEiBGmvr7eGGPMk08+aYqLi4O5hg0bZrZv396hTFVVVeauu+4yJ06cMMZcXF779+8P+fg3tfV5af75jhw5Yowxpr6+3kycONFs2rSpQ3nz8/PNrFmzzPnz540xxvj9fvP222+3+dksKioyL7/8csjnZ8+ebR555BEzffp0x7nCLa/WltWWLVvMxIkTg78P9fX15rPPPruin+GPf/yjmTJliuO8xhjj2vBCbm5ucGvwyy+/VF1d3SVbfd/97nd10003aePGjW69ZbsdP35cW7du1XPPPadOnTpJkjIyMpSXl6cVK1ZIktLT0zVq1Ci98cYbVy1nazIyMtS9e3dVVVVd1Rzh1nOzhIQE3XnnnfrXv/7V6ny+9a1v6aabbtKRI0f03nvvqWfPnpo+fboSEy/+SqampmrixIkdzhtunRcXF2vw4MHBrV5J+uUvf6kePXro3XffbTG/1NRUfe9739ORI0c6lOvrr79WcnKyevbsKeni8howYEDIx9sjIyMjuLfj8/k0YMAAnThxwnHWL774Qtu3b9cLL7ygbt26SZKSk5M1YcIEXXPNNVqyZImys7OVnZ2tJUuWKBAIhJ3n2bNntWfPHhUWFurjjz/WqVOnHGVzsryqqqqUlpYmn88n6eIyuuWWW67o/Wpra9WjRw9HWZu5VrqDBw9WZWWlzp07p02bNumBBx5o8Zp58+Zp1apVunDhgltv2y6VlZX69re/HVxBzb7//e+rsrIy+P+8vDxt3LhRX331le2IIe3du1dpaWlhd2sOHTqknJyc4L/mPyZuuZL1LF3cpf/zn/8cHE663Oeff67Dhw+rX79+2r9/v26//XZXczYLt84rKys1aNCgFtMNGjTokt+JZlVVVdq3b1+7i/By/fv31+23366srCw9/vjjWrt2rc6ePRvycadOnz6tbdu2KSsry/E8Pv30U33nO99ptWzWr1+vAwcOqLS0VKWlpfr000+1fv36sPN89913lZWVpWuvvVYjRozQpk2bHGVzsrzGjBmjQ4cOaeTIkSooKFBZWZkaGxtDvn7z5s3KyclRVlaWiouLNWvWLEdZm7lWugkJCRo9erTKy8tVXl6u7OzsFq/p16+f7rzzThUXF7v1tu1i2rih2jfHu6699lqNHz9eK1eutBGrTY8//rjuu+8+TZo0SXPmzAn+dQ7l5ptvVllZWfBfR39BLhduPe/evVs5OTn68Y9/rDNnzlyyBSlJL7zwgnJycvTrX/9azz//vPr27etqvsuFW+dX+jvxu9/9Tjk5OcrLy9PUqVOVmZnZoVyJiYlauXKliouLddddd2nnzp26//77VVNT0+rj1dXV7X6P2tpa5eXl6dFHH+3wH4lQPvzwQ+Xm5gaPPTz44IP68MMPw05XWlqq3NxcSZfuPbVXqOXY1vJKT09XeXm5Fi9erD59+mj16tWaPn16yNc/8MADKisr044dOzR//nzNmzfPUdZg5g5NfZnc3FwVFRUpIyNDaWlprb5mzpw5Wrt2rc6fP+/mW1+RjIwMHT16tMUK+etf/6of/OAHlzw2depUbd++XUePHrUZsYWioiJt27ZNy5cvV0FBgb7++uurmkdqez1nZmaqrKxMO3fuVEJCgl555ZVLnn/66adVVlamDRs2BAt74MCB+sc//hGRrOHWeb9+/fS3v/2txXR///vfL/mdmDZtmsrKylRaWho8+OdWvp/97Gdas2aNUlNT9dFHH7X5eGsWLVoU3LM5fPiwJOnf//63ZsyYoSFDhujRRx/tUMYBAwbon//8p86dO9eh+TT75JNP9Pnnn2vBggUaPny4nnjiCR07dkx79+51PM/2LC/p4vDID3/4Q02fPl3FxcXatWuXqqurW12W3zRq1CgdOHBAZ86ccZzV1dK98cYbNW/ePM2cObPN19x3331666233HzrK9K7d2+NGjVKzz33nOrr6yVd3P186623NHfu3Etem5qaqsmTJ2vVqlXWc7Zm9OjRGjJkiF5//fWrHeWK1nO3bt20aNEirVu3LuwwzU9/+lOdOXNGv//974NbnrW1tVe0mxpOuHU+adIkVVRUaOvWrcFp1q5dq06dOunee+/t8PuHUlVVpY8//jj4/5MnT+rMmTPq1atXq4/37t075LyeffbZ4J5N3759VV9frxkzZmjQoEGaM2dOh7P26dNHw4cP18KFC1VbWytJCgQCeueddzR48GBt3rxZfr9ffr9fmzdvDrsXUFJSoqlTp+qDDz4I/ps9e7ZKSkranS3UcmxreX3yySc6duxY8P/79+9Xjx491L179xbL8nIVFRXq2bNni+Gq9nDllLFvmjBhQtjXzJw5s8UYTk1NjX7yk58E/9+3b1+tXbvW7Xh69tlntXz5co0ZM0YJCQmqqqrShg0bdNttt7U4ZW3SpEn6wx/+0GIeo0aNCu56du7cWdu2bXM9Z2vy8/P14IMP6rHHHgv5muYx3Wbp6ekROSh4Jeu5f//+wYOSCxYsCPm6Ll26qLi4WMuWLdM999yjrl27Kjk5WQ8//LArWdta55L05ptvaunSpSosLNT58+d1yy236M0331Rysusfj6DGxka9+uqrOn78uK655ho1NTVp7ty5uv766/XMM8+0eLw9wwMbN27URx99pOrqau3atUvSxd/ZvLw8x3lffvllvfbaaxo7dqxSUlLU1NSku+++W/PmzdPx48eDQwVDhw695DS/t99+O3iaoyRNmTJF5eXlWrdu3SXzz87O1v3336+nn35aXbp0ueJcoZZjW8vr7NmzWrRokWpra+Xz+dS5c2e99tprwYO4l9u8ebN2794tY4ySk5P1yiuvhHztlYjrb47w+/1auHChTp48qdWrVwePbiN2hVvnhw4d0syZMzVlyhRXzhEGLhfXpQsAtnEZMABYROkCgEWULgBYROkCgEWULgBYROkCgEWULgBY9P9XUA8oTURYdwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set()\n",
    "\n",
    "\n",
    "tasks = ['MNLI', 'QNLI', 'RTE', 'MRPC', 'QQP', 'SST-2', 'CoLA', 'STS-B']\n",
    "\n",
    "heterogeneous = np.array([patterns[task]['attention']['other'] * 100 for task in tasks])\n",
    "block =  np.array([patterns[task]['attention']['block'] * 100 for task in tasks])\n",
    "diagonal = np.array([patterns[task]['attention']['diagonal'] * 100 for task in tasks])\n",
    "vertical = np.array([patterns[task]['attention']['vertical'] * 100 for task in tasks])\n",
    "vertical_diagonal = np.array([patterns[task]['attention']['mix'] * 100 for task in tasks])\n",
    "\n",
    "wn_hetero = np.array([patterns[task]['weight normed']['Heterogeneous'] * 100 for task in tasks])\n",
    "wn_block =  np.array([patterns[task]['weight normed']['Block'] * 100 for task in tasks])\n",
    "wn_diagonal = np.array([patterns[task]['weight normed']['Diagonal'] * 100 for task in tasks])\n",
    "wn_vertical = np.array([patterns[task]['weight normed']['Vertical'] * 100 for task in tasks])\n",
    "wn_vertical_diagonal = np.array([patterns[task]['weight normed']['Vertical + Diagonal'] * 100 for task in tasks])\n",
    "N = len(heterogeneous)\n",
    "\n",
    "ind = np.arange(N)    # the x locations for the groups\n",
    "width = 0.35       # the width of the bars: can also be len(x) sequence\n",
    "\n",
    "\n",
    "stuff_to_stack = [heterogeneous, block, diagonal, vertical, vertical_diagonal, wn_hetero, wn_block, wn_diagonal, wn_vertical, wn_vertical_diagonal]\n",
    "ps = []\n",
    "summed = None\n",
    "for i, stuff in enumerate(stuff_to_stack):\n",
    "    p = plt.bar(ind, stuff, width, bottom=summed)\n",
    "    ps.append(p)\n",
    "    if summed is None:\n",
    "        summed = stuff\n",
    "    else:\n",
    "        summed += stuff\n",
    "    \n",
    "plt.xticks(ind, tasks)\n",
    "plt.yticks([], [])\n",
    "plt.axhline(100, color='black')\n",
    "# legend = [\"heterogeneous\", \"block\", \"diagonal\", \"vertical\", \"vertical+diagonal\"]\n",
    "# plt.legend((ps[0][0], ps[1][0], ps[2][0], ps[3][0], ps[4][0]), ('heterogeneous', \"block\", \"diagonal\", \"vertical\", \"vertical+diagonal\"))\n",
    "#plt.legend((ps[5][0], ps[6][0], ps[7][0], ps[8][0], ps[9][0]), ('heterogeneous', \"block\", \"diagonal\", \"vertical\", \"vertical+diagonal\"))\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAI2CAYAAAB+Et6iAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdfVzN9/8/8Een00kUWWpjtmEWYoguqEaaa5H4IBMzLOWDEJvLjN9mwzAZYoytz+ZiSo1cbD77YObq62rDTMKQJumCUs451ev3Rx/no3X9OufopMf9dnO7Oe9z3q/3s/d51eP9fr2vzIQQAkRERJWkqOoCiIioemKAEBGRFAYIERFJYYAQEZEUBggREUlhgBARkRQGCD2zwsPDsXr16qoug+iZxQAhgxo5ciRcXV2h0WiKTPfx8cHRo0d1r5OSktCiRQvk5eUZZLkxMTEYPnx4kWkLFy7EP//5T4O0/6RVq1ahdevWcHZ2houLCwICAnD27FmpGmfOnIkVK1YYvEaip4EBQgaTlJSEU6dOwczMDP/+97+ruhyj6tOnD86ePYtjx46hQ4cOmDRpEqrimlxDBTCRDAYIGUxsbCzatWsHf39/xMbG6qbPmDEDycnJCA4OhrOzM7744gsEBgYCAFxdXeHs7Kzbgt+xYwf69OkDV1dXjB07Frdv39a106JFC2zZsgU9e/aEi4sLFixYACEErl69ivnz5+PcuXO6vQKg+Nb99u3b0aNHD7i5uSE4OBgpKSnltl0eCwsL+Pv7IzU1FRkZGVi/fj26d+8OZ2dn9O3bFz/++CMAlFjjtm3bsGvXLmzcuBHOzs4IDg4GAKSkpGDSpEno1KkTfHx88PXXX+uWt2rVKkyePBnTp09Hhw4dsHPnTowcORKfffYZAgIC4OzsjDFjxiA9PR0AoFarMX36dLi7u8PFxQWDBw/GvXv3KvfFEpVGEBlI9+7dxb/+9S9x/vx54eTkJFJTU3XvdevWTfzyyy+617du3RKOjo5Cq9Xqpv3444+ie/fuIjExUWi1WrF69WoxbNgw3fuOjo4iKChI3L9/X9y+fVu4u7uLQ4cOCSGEiI6OFgEBAUXqef/998Xy5cuFEEIcPXpUuLm5iQsXLgi1Wi0WLlwo3nrrrQq1/XcREREiLCxMCCGEWq0Wn3zyiejatasQQog9e/aIO3fuiPz8fBEfHy/atWsnUlJSKlSjEELk5+cLf39/sWrVKqFWq8XNmzeFj4+POHz4sG7ZTk5O4scffxT5+fkiNzdXBAYGijfffFNcu3ZN93rp0qVCCCG2bNkixo8fL3JyckReXp44f/68yMrKKvU7JKoM7oGQQZw6dQrJycno06cP2rRpg5deegm7d++uVBtbt25FUFAQXn31VSiVSgQHB+PSpUtF9kLeffdd1K1bF40aNYK7uzv++OOPCrW9a9cuDB48GK1bt4ZKpcK0adNw7tw5JCUlSbW9b98+uLi4oGvXrrh48SI+//xzAIVDW88//zwUCgX69u2LV155Bb/99luF18H58+eRnp6OiRMnQqVS4aWXXsLQoUOxZ88e3Wfat2+P7t27Q6FQoFatWgCAQYMGoWnTpqhVqxZ69+6NS5cuAQCUSiUyMzNx48YNmJubo02bNrC2tq5wPURlUVZ1AfRsiI2NhaenJ5577jkAgK+vL3bu3InRo0dXuI3k5GQsWrQIixcv1k0TQiAlJQUvvvgiAMDe3l73npWVFR4+fFihtu/evYvWrVvrXtepUwe2trZISUlB48aNK91279698emnnxabHhsbi02bNulCLycnBxkZGRWqEQBu376Nu3fv6obhACA/P7/I6xdeeKHYfH+vPScnBwDg5+eHO3fuYNq0aXjw4AEGDBiAqVOnwsLCosI1EZWGAUJ6e/ToEfbu3YuCggJ4enoCADQaDR48eIA//vgDLVu2LDaPmZlZsWkNGzZEcHAwBgwYUOkaSmrvSQ4ODkX2ZHJycpCZmYnnn3++0ssqze3btzF37lxs3rwZzs7OMDc3h5+fX5k1/n1aw4YN0bhxY/zwww+lLqe8n/VJFhYWmDhxIiZOnIikpCQEBQWhadOmGDJkSIXbICoNh7BIbwcOHIC5uTni4+MRGxuL2NhY7NmzBy4uLrqD6Q0aNMCtW7d08zz33HNQKBRFpgUEBGD9+vW4cuUKACArKwt79+6tUA12dnZISUkpdvrwY76+voiJicGlS5eg0WiwfPlytG3bVrf3YQi5ubkwMzPT7YVFR0frfpbSarSzsysyjNa2bVvUqVMH69evx6NHj5Cfn4+EhIRKDYM96fjx47h8+TLy8/NhbW0NpVIJhYK/9mQY7Emkt507d2LQoEFo1KgR7O3tdf9GjBiBXbt2IS8vD0FBQVi7di1cXFywceNGWFlZITg4GMOHD4eLiwvOnTuHHj16YNy4cZg2bRo6dOgAX19fHD58uEI1dOrUCc2bN4eXlxfc3d2Lve/h4YHQ0FBMmjQJXl5euHXrlsGvv2jevDnGjBmDgIAAeHh4ICEhAR06dCizxn/84x9ITEyEi4sLJkyYAHNzc0RGRuKPP/7Am2++iU6dOmHu3LnIzs6WqunevXuYPHkyOnbsiL59+8LNza3IXhGRPsyE4AOliIio8rgHQkREUhggREQkhQFCRERSGCBERCSFAUJERFIYIEREJMWoV6JnZDxEQYFxzhK2s7NGWprcufFVgfUaF+s1LtZrXMasV6EwQ/36dYzStlEDpKBAGC1AHrdfnbBe42K9xsV6jau61QtwCIuIiCQxQIiISAoDhIiIpDBAiIhICgOEiIikMECIiEgKA4SIiKSY9PNA8rT5UFqYP/V5ZVWnevVdHus17vJYr3GXV53qrYq/ZRVl1AsJ09Ky9bo4xt7eBgvDdkvNG77MF6mpWdLLllGd6tWnVoD1lof1GldNqlffWhUKM9jZWUvPX2bbRmmViIieeQwQIiKSwgAhIiIpDBAiIpLCACEiIikMECIiksIAISIiKQwQIiKSwgAhIiIpDBAiIpJi1FuZ1DRaTR7Cl/lKz0tEz6Zn9W8DA8SALFRK/OI3WGpez7hoA1dDRKbiWf3bwCEsIiKSwgAhIiIpHMIiIjKyfLVGeigqX60xcDWGwwAhIjIyc0sVhm4LkZp3+7C1ANSGLchAGCAG9KxuZRARlYQBYkDP6lYGEVFJTDpAntVzp4mIngUmHSDP6rnTpkCfcH48/9NU3eolqglMOkDIePQJZ+DpB3R1q5eoJuB1IEREJMWk90B4VhMRkeky6QDhWU3Go084P56fiGo2kw4QMh59whlgQBMRj4EQEZEkBggREUlhgBARkRQeAyEyAl74SDUBA4TICHjhI9UEHMIiIiIpDBAiIpLCACEiIikMECIiksIAISIiKQwQIiKSwgAhIiIpvA6EiMjINHma/96AVG5eU8UAISJotfn6XTmvzTdgNc8elVKFax/JXVjabE40TPXO10YNEDs7a2M2Xy57e5sqXX5lsV7jYr1l0/f2/ly/xmOqtRo1QNLSslFQIKTn13elpaZm6TV/ZVWneg3RIVlv6VivcdW0evWpVaEwM9rGPA+iExGRFAYIERFJYYAQEZEUBggREUlhgBARkRQGCBERSWGAEBGRFAYIERFJYYAQEZEUBggREUlhgBARkRQGCBERSWGAEBGRFAYIERFJ4QOlaih9npD2eH4qXb5aA8+4aL3mJzJ1DJAaSp8npAGm/ZQ0U2BuqdL7AU1cv2TqOIRFRERSGCBERCSFAUJERFIYIEREJIUBQkREUhggREQkhQFCRERSeB2IAelzcR4vzCN6dhVoNf+9dkpuXlPFADEgfS7O44V5RM8uhYUK/cPipObdtcwPpvq3gUNYREQkhQFCRERSGCBERCSFAUJERFIYIEREJIUBQkREUhggREQkhQFCRERSGCBERCSFAUJERFIYIEREJIUBQkREUhggREQkxah347Wzs9Zrfn1vj25vb6PX8p821mtcrNe4WK/xmGqtRg2QtLRsFBQI6fnt7W30uj16amqW9LJl6PslP816DdEhWW/pWK9x1bR69alVoTDTe2O+1LaN0ioRET3zGCBERCSFTyQkMgJ9jt89np/I1DFAiIxAn8cbA3zEcXm0mjyEL/PVa37SHwOEiKodC5USv/jJB7RnXLQBq6m5eAyEiIikMECIiEgKh7CIqNrJV2v0GobKV/MkBUNggBBRtWNuqcLQbSHS8xeeIceTFPTFISwiIpLCPRAi4nUrJIUBQkS8boWkcAiLiIikcA+EyAgKtJr/bpXLz09k6hggVC1Ut9M2FRYq9A+Lk55/1zI/cEiITB0DhKoFnrZJZHoYIAakz7AFhyyIqLphgBiQPsMWHLIgouqGZ2EREZEUBggREUlhgBARkRQGCBERSWGAEBGRFAYIERFJYYAQEZEUBggREUlhgBARkRQGCBERSWGAEBGRFAYIERFJYYAQEZEUBggREUlhgBARkRQGCBERSWGAEBGRFD6RkKoFTZ7mv881l5+fiAyLAULVgkqpwrWPBkvPX/isej4ymMiQGCBEVO1wj9Q08BgIEVU7SlG181Mho+6B2NlZG7P5ctnb21Tp8iuL9RoX6zWup11v/7A46Xl3LfODvb3KgNUYl6n2BaMGSFpaNgoK5KNe35WWmpql1/yVVZ3qNUSHZL2lY73GVdPq1adWhcLMaBvzHMIiIiIpDBAiIpLCACEiIikMECIiksIAISIiKQwQIiKSwgAhIiIpJn0rkwKt5r/3MJKbl4iIjMekA0RhoZK+2nTXMj/w5nlEFaPPxtrj+anmMekAIaKnQ5+NNYAbbDUVj4EQEZEUBggREUlhgBARkRQGCBERSWGAEBGRFAYIERFJYYAQEZEUBggREUlhgBARkRQGCBERSWGAEBGRFAYIERFJYYAQEZEUBggREUlhgBARkRQGCBERSWGAEBGRFAYIERFJYYAQEZEUBggREUlhgBARkRQGCBERSWGAEBGRFAYIERFJYYAQEZEUBggREUlhgBARkRQGCBERSWGAEBGRFKUxG7ezszZm8+Wyt7ep0uVXFus1LtZrXKzXeEy1VqMGSFpaNgoKhPT8+q601NQsveavrOpUryE6JOstHes1rppWrz61KhRmRtuY5xAWERFJMeoeCJGhFGg1aDYnWq/5iciwGCBULSgsVOgfFic9/65lfgDUhiuIiDiERUREchggREQkhQFCRERSGCBERCSFB9FrKJ7VRET6YoDUUDyriYj0xSEsIiKSwgAhIiIpDBAiIpLCACEiIikMECIiksIAISIiKQwQIiKSwgAhIiIpDBAiIpLCACEiIikMECIiksIAISIiKQwQIiKSwgAhIiIpDBAiIpLCACEiIikMECIiksIAISIiKQwQIiKSwgAhIiIpDBAiIpLCACEiIikMECIiksIAISIiKQwQIiKSwgAhIiIpDBAiIpLCACEiIilKYzauUJjp3YZDfasqXX5lVad69akVYL3lYb3GVZPq1adWY/6cZkIIYbTWiYjomcUhLCIiksIAISIiKQwQIiKSwgAhIiIpDBAiIpLCACEiIikMECIiksIAISIiKQwQIiKSYtRbmVSGj48PNBoNDh06BHNzcwBATEwMZs2ahXnz5qF27dqYNWsWVqxYgb59++reP3jwICIiIpCUlITBgwfjxIkTxdpu0aIFzpw5gzp16hi0Zo1Gg+XLl+PAgQNQKpWwtLREcHAw+vTpgxMnTmDUqFEICwtDUFAQAODEiRNYvHgxYmJiyqzLx8cHkZGRcHR0NGi9T7avUqmgUqmg1Wrh7++P+Ph4AEBOTg7u3r2LJk2aAAC8vb3h4eGBoKAg3bTHtS9ZssQotZXXDxYtWoQXX3wRWq0WjRs3xkcffQR7e3sAwMiRI5GcnAxra2uo1WoMHz4cb7/9NgAgNTUVS5cuxenTp1G3bl2Ym5tj6NChGDp0qMHqL6tPAEBCQgIWL16MmzdvIj8/H61atcLMmTPx0ksvAQBmzpyJo0ePon79+nj06BF69OiB6dOnG6y+vXv3Yt26dRBCQK1Wo3Xr1li2bFmp04cMGQKNRgOtVos///wTr732GgDAyckJH3/8sa7dM2fOYMmSJXjw4AEAoGvXrnjvvfdgZqb/bTS0Wi3WrFmDPXv2QKVSwdzcHJ06dUJYWBgsLCxKnGfVqlXIycnB+++/X+L7hw4dQlBQED7//HP06NFD7xofK2k93rx5s9R1uGjRIkREROCHH36Aubk58vLyMGTIELzzzjsl/kzffvstHBwcIISAlZUVFixYgJYtWxqs/koTJqJbt27C399fHDx4UDctMDBQ+Pv7i6ioKBEdHS26desmevbsKbRarRBCiOjoaDFp0iQhhBC3bt0Sbm5uJbbt6OgosrOzDV7zrFmzRGhoqHj06JEQQojLly8LLy8vcfToUXH8+HHh6ekpPDw8xP3794UQQhw/flz4+/uXW1e3bt3E5cuXDV5vSe1fvnxZtG7dWty5c6fEGkubZszayusHj7/zgoICMWXKFBEeHl7ksz/99JMQQojk5GTRoUMHcenSJZGTkyN69uwpIiMjRX5+vhBCiAcPHogtW7YYtP6y+kRmZqbw8PAQe/bs0X1+06ZNokePHkKtVgshhHj//fdFVFSUrr5u3bqJAwcOGKS2lJQU4e7uLpKTk4UQhevv4sWLpU5/Ulm/X49/zuvXrwshhFCr1SIgIEDs3LnTIHWHhYWJiRMniqysLCGEEFqtVmzdurXM3+mIiAjxySeflPr+pEmTxKhRo8T48eMNUqMQpa/fx0pah3v27BEBAQG6/qJWq8WVK1dKbP/vP9O//vUvMXbsWIPVL8OkhrD8/f11W+e3bt1CTk5Oka3wNm3aoGnTptixY0dVlahz+/Zt7N27Fx988AEsLS0BAI6OjggJCcHnn38OAHBwcEDv3r3xxRdfVGWpZXJ0dETdunWRkpJS1aXolNcPHjMzM4Orqyv++uuvEttp2LAhmjZtiuvXr2P37t2wtbXF+PHjoVAUdnsbGxsEBAQYrO7y+kRUVBTc3Nx0eyMAMHr0aNSrVw/ff/99sfZsbGzw+uuv4/r16wap7969e1AqlbC1tQVQuP6cnJxKnV4Zjo6Ouj1UlUoFJycnJCcn613zn3/+iQMHDuDDDz+EtbU1AECpVGLYsGGoVasWFi9eDF9fX/j6+mLx4sXIz88vt82MjAwcP34cy5Ytw9mzZ5Gamqp3nUDp67csKSkpqF+/PlQqFYDCdde8efMKLS87Oxv16tXTr2g9mVSAuLm5ISEhAffv38fOnTsxcODAYp+ZOnUq1q5di0ePHlVBhf+TkJCAl19+WddZHmvfvj0SEhJ0r0NCQrBjxw7cvXv3aZdYIadPn0b9+vXL3Q2+evUq/Pz8dP8eh6QxVKQfAIXDRYcPH9YNaf5dYmIirl27hhYtWuDixYto27at0WoGyu8TCQkJaNeuXbH52rVrV6TPPJaSkoIzZ85U+o95aVq2bIm2bdvC29sbkydPxubNm5GRkVHqdFlpaWnYv38/vL299a75999/xyuvvFLiH8pt27bh0qVLiImJQUxMDH7//Xds27at3Da///57eHt7o0GDBujRowd27typd51A6eu3LH379sXVq1fRs2dPzJo1C3FxccjLyyv187GxsfDz84O3tzeioqIwceJEg9Quy6QCxMzMDH369EF8fDzi4+Ph6+tb7DMtWrSAq6sroqKiqqDC/xFl3MT4yXHfBg0aYOjQoVizZs3TKKvCJk+ejF69eiEwMBChoaG6LaDSvPrqq4iLi9P9M2bHLa8fHD16FH5+fujcuTPS09OLbNEDwIcffgg/Pz/Mnj0bCxcuRLNmzYxW65PK6xMV7TPr16+Hn58fQkJCMG7cOHh4eBikPoVCgTVr1iAqKgru7u44dOgQBgwYgAcPHpQ4PTMzs9LLyM7ORkhICMaMGWOw4CvNsWPH4O/vrzueN2jQIBw7dqzc+WJiYuDv7w+g6N6uvkpbv2WtRwcHB8THx+Pjjz9GkyZNEBkZifHjx5f6+YEDByIuLg4HDx7E9OnTMXXqVIPULsukAgQo/EIjIiLg6OiI+vXrl/iZ0NBQbN68GVlZWU+5uv9xdHTEzZs3i3WOc+fOwdnZuci0cePG4cCBA7h58+bTLLFMERER2L9/P5YvX45Zs2bh3r17VV1SEWX1Aw8PD8TFxeHQoUMwMzPDypUri7w/d+5cxMXFYfv27brwad26Nc6fP2/UmsvrEy1atMCvv/5abL7ffvutSJ8JCgpCXFwcYmJidCcAGLrOESNGYNOmTbCxscHJkyfLnF6SBQsW6PZGr127BgDIzc1FcHAwPD09MWbMGIPU6uTkhBs3buD+/fsGae/ChQtITEzEnDlz4OPjgxkzZiApKQmnT582SPtA5dYjUDgk5+LigvHjxyMqKgpHjhxBZmZmiev4Sb1798alS5eQnp5usNory+QC5KWXXsLUqVMxYcKEMj/Tq1cvfPXVV0+xsqIaN26M3r1744MPPoBarQZQOITx1VdfYcqUKUU+a2Njg3feeQdr166tilLL1KdPH3h6emLdunVVXUoRFekH1tbWWLBgAbZs2VLuEGG/fv2Qnp6ODRs26PYEsrOzKzTkUVHl9YnAwECcOHECe/fu1c2zefNmWFpaonv37garozQpKSk4e/as7vWdO3eQnp4OOzu7Eqc3bty41Lbmz5+v2xtt1qwZ1Go1goOD0a5dO4SGhhqs5iZNmsDHxwfh4eHIzs4GAOTn5+O7776Dm5sbYmNjodVqodVqERsbW+7eWnR0NMaNG4effvpJ92/SpEmIjo7Wu9bS1m9Z6/HChQtISkrSvb548SLq1auHunXrFlvHf3fixAnY2toWGzJ9mkzmNN4nDRs2rNzPTJgwodjY5YMHD9ClSxfd62bNmmHz5s2GLk9n/vz5WL58Ofr27QszMzOkpKRg+/btaNWqVbHTiQMDA/H1118Xa6N379664QsrKyvs37/faPWWJiwsDIMGDcK7775b6mceHwN5zMHBwegnB1SkH7Rs2VJ3osKcOXNK/Vzt2rURFRWFpUuX4s0330SdOnWgVCrx1ltvGbLkMvsEAHz55ZdYsmQJli1bhqysLDRv3hxffvkllErj/yrm5eVh1apVuH37NmrVqoWCggJMmTIFL7zwAubNm1dsemWGoHbs2IGTJ08iMzMTR44cAVDYt0NCQvSu+5NPPsHq1asxePBgWFhYoKCgAF27dsXUqVNx+/Zt3XCUl5dXkVOyt27dqjs9HQDGjh2L+Ph4bNmypUj7vr6+GDBgAObOnYvatWtL11na+i1rPWZkZGDBggXIzs6GSqWClZUVVq9erTvR4+9iY2Nx9OhRCCGgVCqxcuXKUj/7NPCJhAai1WoRHh6OO3fuIDIyUncWDtVc5fWJq1evYsKECRg7dqxBr0UheloYIEREJMXkjoEQEVH1wAAhIiIpDBAiIpLCACEiIikMECIiksIAISIiKQwQIiKSwgAhIiIpDBAiIpLCACEiIikMECIiksIAISIiKQwQIiKSwgB5xnz//fcVfhpcTEwMhg8fbuSKTIuPjw+OHj1a1WVUWGW+T6KnjQFiAtatW4dx48YVmdazZ88Spz35gJySDBgwAF9++aVB6ho5ciS+++67Ut9PSkpCixYtij2Iavr06Vi1apVBaqgKq1atKvHxszNnzsSKFSuKTDNkID1en3l5ebpphvw+n3TixAm0bNkSzs7OcHZ2Rq9evSr0VL6SaqyJGyJUiAFiAlxcXHD27Fnk5+cDAO7evYu8vDxcunSpyLQbN27AxcWlKkst0W+//YYzZ87o3c6Tf5SqihACsbGxsLW1RWxsbFWXY1QODg44e/Yszpw5gxkzZmDevHlITEx86nWYwvdOchggJuD111/XBQYAnDp1Cu7u7mjatGmRaS+//DKef/55ZGVlYfbs2fDy8sIbb7yBFStW6ILm71uDR44cQa9evdCxY0d88MEHCAwMLLZXsXjxYri6usLHxweHDh0CAKxYsQKnTp3CwoUL4ezsjIULF5Za/9ixY4ttmT9p+/bt6NGjB9zc3BAcHIyUlBTdey1atMA333yDnj17omfPnjhx4gS6dOmCL774Ap07d4aXlxcOHDiAQ4cOoVevXnBzc0NkZKRu/oKCAqxfvx7du3eHu7s7QkNDkZmZqXs/NjYW3bp1g7u7e4WeSX/q1CmkpqZizpw52LNnDzQaDQBg27Zt2LVrFzZu3AhnZ2cEBwdjxowZSE5ORnBwMJydnXWP+D137hwCAgLg4uKCAQMGFHm88ciRI/HZZ58hICAAzs7OGDNmDNLT0wEUPvYYAFxdXeHs7IyzZ88W+z7PnDmDwYMHo2PHjhg8eHCR4C6r7bKYmZmhe/fuqFu3LhITE3Hw4EEMHDgQHTp0QNeuXYvsTZZU4/z583Hu3Dk4OzvrNnA0Gg0WL14Mb29veHh4IDw8HI8ePQIA3Xe8fv16eHp6YtasWVi1ahVCQ0Px3nvvwdnZGf369cP58+d1y12/fj3eeOMN3d7SsWPHyv256CkQZBICAwPFpk2bhBBCLFiwQHz33Xdi+fLlRabNnDlTCCHEhAkTxLx588TDhw/FvXv3xODBg8WWLVuEEEJER0eLgIAAIYQQaWlpwtnZWezfv19otVqxefNm4eTkJLZv3677rJOTk9i2bZvIy8sT33zzjfD09BQFBQW6mh5/tiS3bt0Sjo6OIisrS3h5eYlffvlFCCFEWFiYiIiIEEIIcfToUeHm5iYuXLgg1Gq1WLhwoXjrrbd0bTg6OorRo0eLjIwMkZubK44fPy5atWolVq1aJTQajdi2bZtwd3cX06ZNE1lZWSIhIUG8/vrr4ubNm0IIITZv3iyGDBki/vrrL6FWq8W8efPE1KlThRBCXLlyRbRv316cPHlSqNVqsWjRItGqVStdnSWZNWuWmDx5stBoNMLNzU3s27dP9977778vli9fXuTz3bp1K9LenTt3hJubmzh48KDIz88XR44cEW5ubiItLU23Tt98801x7do1kZubKwIDA8XSpUuLrE+tVqtr78nvMyMjQ7i4uIidO3cKrUc2RhEAACAASURBVFYrdu3aJVxcXER6enq5bf/d8ePHxRtvvCGEECI/P1/88MMPwsnJSVy9elUcP35c/PHHHyI/P19cunRJdO7cWfz4448VqvGxjz76SIwfP15kZGSIrKwsMX78ePHpp5/qlt2qVSuxZMkSoVarRW5uroiIiBBt2rQRBw8eFHl5eeLTTz8VQ4YMEUIIcfXqVdGlSxdx584dXQ03btwo9Tukp4d7ICbCzc0N//d//wegcCvYxcUFHTt2LDLNzc0N9+7dw6FDhzB79mzUrl0bdnZ2GD16dInHRg4fPozXXnsNPXv2hFKpxKhRo9CgQYMin2nUqBGGDh0Kc3Nz+Pv7IzU1Fffu3atU7bVq1UJwcDA+++yzYu/t2rULgwcPRuvWraFSqTBt2jScO3cOSUlJus8EBQXB1tYWtWrVAgAolUqEhITAwsICffv2RUZGBkaNGgVra2u89tpraN68OS5fvgwA2Lp1K6ZOnYoXXngBKpUKEydOxP79+5GXl4d9+/bB29sbrq6uUKlUCA0NhUJRepfPzc3Fvn370L9/f1hYWKBXr16VHsaKi4tDly5d0LVrVygUCnh6eqJNmza6PTsAGDRoEJo2bYpatWqhd+/eur3M8hw8eBCvvPIKBg4cCKVSCV9fXzRr1gz/+c9/pNq+e/cuXFxc0KlTJ3z++edYsmQJmjVrBnd3d7Ro0QIKhQItW7ZEv379cPLkyQqvAyEEtm/fjtmzZ8PW1hbW1tYYP358kT6qUCgwefJkqFQq3ffesWNHdO3aFebm5vDz88Mff/wBADA3N4dGo8HVq1eh1WrRuHFjvPzyyxWuh4xHWdUFUCEXFxd88803yMzMRHp6Opo0aYIGDRpg5syZyMzMxJUrV+Di4oLk5GTk5eXBy8tLN29BQQEaNmxYrM27d+/ihRde0L02MzMr8hpAkUCxsrICAOTk5FS6/iFDhmDjxo346aefitXQunVr3es6derA1tYWKSkpaNy4MQAUq93W1hbm5uYAoPvjYmdnp3vf0tISDx8+BAAkJyfjn//8Z5FgUCgUSEtLK/bz165dG7a2tqX+DD/++COUSiW6dOkCAOjfvz/eeecdpKen47nnnqvQekhOTsa+ffuK/FHPy8uDu7u77rW9vb3u/1ZWVhVe33fv3kWjRo2KTGvUqFGRIcHKtO3g4IDDhw8Xm/7rr7/i008/xZUrV6DVaqHRaNC7d+8K1QgA6enpyM3NxaBBg3TThBAoKCjQva5fvz4sLS2LzPdkX6xVqxbUajXy8vLwyiuvYPbs2Vi1ahUSExPh5eWFmTNn4vnnn69wTWQcDBAT4ezsjOzsbGzfvh0dOnQAAFhbW8PBwQHbt2+Hg4MDXnrpJVhaWkKlUuH48eNQKsv++uzt7Yv8cRFC4M6dO0ap//HW/8qVK9G8eXPddAcHB9y+fVv3OicnB5mZmUV++c3MzKSX+8ILL2DRokXo2LFjsfccHBxw9epV3evc3Nwix0f+LjY2Fjk5OejWrRuAwvWl1Wqxa9cuvP322xWqs2HDhvDz88OHH35Y6Z+lvPYdHByQnJxcZNpff/2FN954o9LLKktYWBgCAwOxYcMGWFpa4qOPPkJGRkapNf59Wv369VGrVi3Ex8eX+ke+st95//790b9/f2RnZyM8PByffvopli5dWqk2yPA4hGUiatWqhTZt2mDz5s1FzrTq2LFjkWkODg7w9PTEJ598guzsbBQUFODmzZslDjF07doVly9fxoEDB5CXl4dvvvmmUsNTDRo0wK1btyr8eT8/P6jVahw5ckQ3zdfXFzExMbh06RI0Gg2WL1+Otm3b6vY+9DV8+HB89tlnupBKT0/HgQMHAAC9evXCwYMHcerUKWg0GkRERBTZCn5SSkoKjh07hsjISMTGxiI2NhZxcXF49913ERcXB6BwL+jJoTeg+DoaMGAA/vOf/+Dnn39Gfn4+1Go1Tpw4UaHgfu6556BQKEpd5127dsWff/6JXbt2IS8vD3v27EFiYiK8vb3LbbsyHj58iHr16sHS0hK//fYbdu/eXWaNdnZ2SElJ0Z1woFAoMGTIECxatAhpaWkACtfvzz//LFXPtWvXcOzYMWg0GqhUKlhaWpY5FElPD78FE+Lq6oq0tLQiW9MdO3ZEWloaXF1dddOWLFkCrVaLvn37wtXVFZMnT0Zqamqx9p577jmsXLkSS5cuhbu7OxITE9GmTRtYWFhUqJ5Ro0Zh//79cHV1rdAWtbm5OSZPnlxkK9/DwwOhoaGYNGkSvLy8cOvWrTLP2KqsUaNGwcfHB2PGjIGzszOGDh2K3377DQDw2muvITw8HNOnT8cbb7yBunXrFhvCeywuLg6tWrWCl5cX7O3tdf9GjhyJy5cvIyEhAf/4xz+QmJgIFxcXTJgwAUDh8Zu1a9fCxcUFGzduRMOGDbFmzRqsW7cOnTt3RteuXbFx48ZSg+tJVlZWCA4OxvDhw+Hi4oJz584Veb9+/fqIjIzEpk2b4O7ujg0bNiAyMrLCw2sVNX/+fERERMDZ2RmrV69Gnz59yqyxU6dOaN68Oby8vHRDdTNmzMArr7yCoUOHokOHDhg9ejSuX78uVY9Go8GyZcvg7u4OLy8vpKenY9q0aQb5WUk/ZkIIUdVF0NNRUFCALl264NNPP0WnTp2quhwiqua4B/KM+/nnn/HgwQNoNBrd9RPt27ev4qqI6FnAg+jPuHPnzmH69OnQaDRo3rw5Vq9erTuziYhIHxzCIiIiKRzCIiIiKQwQIiKSwgAhIiIpRj2InpHxEAUFxjnEYmdnjbS0bKO0bQys17hYr3GxXuMyZr0KhRnq169jlLaNGiAFBcJoAfK4/eqE9RoX6zUu1mtc1a1egENYREQkiQFCRERSGCBERCSFAUJERFIYIEREJIUBQkREUhggREQkxaRvppinzYfSwvypzyurOtWr7/JYr3GXx3qNu7zqVG9V/C2rKKNeSJiWlq3XxTH29jZYGLa7/A+WIHyZL1JTs6SXLaM61atPrQDrLQ/rNa6aVK++tSoUZrCzs5aev8y2jdIqERE98xggREQkhQFCRERSGCBERCSFAUJERFIYIEREJIUBQkREUhggREQkhQFCRERSGCBERCSFAUJERFIYIEREJIUBQkREUhggREQkhQFCRERSGCBERCSFAUJERFIYIEREJIUBQkREUoz6THQiImPQavIQvsxXr/lJfwwQIqp2LFRK/OI3WHp+z7hoA1ZTc3EIi4iIpHAPhIjIyPQZcjPl4TYGCBGRkekz5GbKw20cwiIiIikMECIiksIhLCKqdvLVGr2GdvLVGgNWU3MxQIio2jG3VGHothDp+bcPWwtAbbiCaigGCBGRkemzx2TKe0sMECIiI9Nnj8mU95ZMOkCe1XOniYieBSZ9FpZCFFTJvEREVD6T3gOpbrt93GMioprEpAOkunlWrzalZx/vbksyGCBExLvbkhSTPgZCRESmi3sgRMQru0kKA4SIeGU3STFqgNjZWRuz+XLZ29tU6fIri/Ua19OsV6vN1++gtDaf69fIqlO9plqrUQMkLS0bBQVCen59V1pqapZe81dWdarXEB2S9ZbO3t5G74PSXL+lq2n16lOrQmFmtI15HkQnIiIpDBAiIpLCACEiIikMECIiksLTeGso3rqCiPTFAKmh9L1b8dO+2zEDj8j0MEBqqOp24Rjv1URkengMhIiIpDBAiIhICgOEiIik8BiIAelzR1PezZSIqhsGiAFVt0fwEhHpg0NYREQkhQFCRERSOIRFRGRkmjzNf4ep5eY1VQwQIiIjUylVuPaR3IWwzeZEw1SPj3IIi4iIpDBAiIhICgOEiIikMECIiEgKA4SIiKQwQIiISAoDhIiIpDBAiIhICgOEiIikMECIiEgKA4SIiKQwQIiISAoDhIiIpDBAiIhICgOEiIikMECIiEgKA4SIiKTwiYQ1lD6P2Hw8PxHVbAyQGkqfR2wCpv2YTao8blCQDAYIEXGDgqTwGAgREUlhgBARkRQGCBERSTHqMRA7O2tjNl8ue3ubKl1+ZbFe42K9xsV6jcdUazVqgKSlZaOgQEjPr+9KS03N0mv+yqpO9RqiQ7Le0rFe46pp9epTq0JhZrSNeQ5hERGRFAYIERFJYYAQEZEUBggREUlhgBARkRQGCBERSWGAEBGRFAYIERFJYYAQEZEUBggREUlhgBARkRQGCBERSWGAEBGRFAYIERFJYYAQEZEUoz4PhMhQ8tUaeMZF6zU/ERkWA4SqBXNLFYZuC5Gef/uwtQDUhiuIiEw7QDR5mv/+4svNS0RExmPSAaJSqnDto8FS8zabEw1ucRIRGQ8PohMRkRQGCBERSWGAEBGRFAYIERFJYYAQEZEUBggREUlhgBARkRSTvg6EjKdAq/nvtTLy8xNRzcYAqaEUFir0D4uTnn/XMj/wQk2imo0BQmQEvPkj1QQMECIj4M0fqSbgQXQiIpLCPRAiqnb0uVP34/mfJn1OWjHlE1YYIERU7ehzp27g6d+tW5+TVkz5hBUOYRERkRQGCBERSWGAEBGRFAYIERFJYYAQEZEUBggREUkx6mm8dnbWxmy+XPb2NlW6/MpivcbFeo2L9RqPqdZq1ABJS8tGQYGQnl/flZaamqXX/JVVneo1RIdkvaVjvcZV0+rVp1aFwsxoG/McwiIiIikMECIiksJbmRiQPvfnedr35iEi0hcDxID0uT/P0743DxGRvjiERUREUhggREQkhQFCRERSGCBERCSFAUJERFIYIEREJIWn8RJRtaPPM8Yfz0/6Y4AQUbWjzzPGAdN+znh1wgChakGfq/wfz09EhsUAoWpBn6v8AV7pT2QMPIhORERSGCBERCSFAUJERFIYIEREJIUBQkREUhggREQkhQFCRERSeB0IkRHwwkeqCRggVC1Ut3sf8cJHqgkYIFQt8N5HRKaHAWJA+mwl8+6gRFTdMEAMSJ+tZG4hE1F1wwAhomp3jIlMAwOEiHiMiaTwOhAiIpLCACEiIikMECIiksIAISIiKQwQIiKSwgAhIiIpDBAiIpJi0teB8NYgRESmy6QDhLcGISIyXRzCIiIiKUbdA7GzszZm8+Wyt7ep0uVXFus1LtZrXKzXeEy1VqMGSFpaNgoKhPT8+q601NQsveavrOpUryE6JOstnZ2tpd43J0zLfHpDsNVt/da0evWpVaEwM9rGvEkfAyGqrnhzQqoJeAyEiIikMECIiEgKA4SIiKQwQIiISAoDhIiIpDBAiIhICgOEiIikMECIiEgKA4SIiKQwQIiISAoDhIiIpDBAiIhICgOEiIikMECIiEgKA4SIiKQwQIiISAoDhIiIpDBAiIhICgOEiIikMECIiEgKA4SIiKQwQIiISAoDhIiIpDBAiIhICgOEiIikMECIiEgKA4SIiKQwQIiISAoDhIiIpDBAiIhICgOEiIikMECIiEgKA4SIiKQwQIiISAoDhIiIpDBAiIhICgOEiIikKI3ZuEJhpncbDvWtqnT5lVWd6tWnVoD1lof1GldNqlefWo35c5oJIYTRWiciomcWh7CIiEgKA4SIiKQwQIiISAoDhIiIpDBAiIhICgOEiIikMECIiEgKA4SIiKQY9Ur0yvDx8YFGo8GhQ4dgbm4OAIiJicGsWbMwb9481K5dG7NmzcKKFSvQt29f3fsHDx5EREQEkpKSMHjwYJw4caJY2y1atMCZM2dQp04dg9as0WiwfPlyHDhwAEqlEpaWlggODkafPn1w4sQJjBo1CmFhYQgKCgIAnDhxAosXL0ZMTEyZdfn4+CAyMhKOjo4GrffJ9lUqFVQqFbRaLfz9/REfHw8AyMnJwd27d9GkSRMAgLe3Nzw8PBAUFKSb9rj2JUuWGKW28vrBokWL8OKLL0Kr1aJx48b46KOPYG9vDwAYOXIkkpOTYW1tDbVajeHDh+Ptt98GAKSmpmLp0qU4ffo06tatC3NzcwwdOhRDhw41WP1l9QkASEhIwOLFi3Hz5k3k5+ejVatWmDlzJl566SUAwMyZM3H06FHUr18fjx49Qo8ePTB9+nSD1bd3716sW7cOQgio1Wq0bt0ay5YtK3X6kCFDoNFooNVq8eeff+K1114DADg5OeHjjz/WtXvmzBksWbIEDx48AAB07doV7733HszM9L8KWqvVYs2aNdizZw9UKhXMzc3RqVMnhIWFwcLCosR5Vq1ahZycHLz//vslvn/o0CEEBQXh888/R48ePfSu8bGS1uPNmzdLXYeLFi1CREQEfvjhB5ibmyMvLw9DhgzBO++8U+LP9O2338LBwQFCCFhZWWHBggVo2bKlweqvNGEiunXrJvz9/cXBgwd10wIDA4W/v7+IiooS0dHRolu3bqJnz55Cq9UKIYSIjo4WkyZNEkIIcevWLeHm5lZi246OjiI7O9vgNc+aNUuEhoaKR48eCSGEuHz5svDy8hJHjx4Vx48fF56ensLDw0Pcv39fCCHE8ePHhb+/f7l1devWTVy+fNng9ZbU/uXLl0Xr1q3FnTt3SqyxtGnGrK28fvD4Oy8oKBBTpkwR4eHhRT77008/CSGESE5OFh06dBCXLl0SOTk5omfPniIyMlLk5+cLIYR48OCB2LJli0HrL6tPZGZmCg8PD7Fnzx7d5zdt2iR69Ogh1Gq1EEKI999/X0RFRenq69atmzhw4IBBaktJSRHu7u4iOTlZCFG4/i5evFjq9CeV9fv1+Oe8fv26EEIItVotAgICxM6dOw1Sd1hYmJg4caLIysoSQgih1WrF1q1by/ydjoiIEJ988kmp70+aNEmMGjVKjB8/3iA1ClH6+n2spHW4Z88eERAQoOsvarVaXLlypcT2//4z/etf/xJjx441WP0yTGoIy9/fX7d1fuvWLeTk5BTZCm/Tpg2aNm2KHTt2VFWJOrdv38bevXvxwQcfwNLSEgDg6OiIkJAQfP755wAABwcH9O7dG1988UVVllomR0dH1K1bFykpKVVdik55/eAxMzMzuLq64q+//iqxnYYNG6Jp06a4fv06du/eDVtbW4wfPx4KRWG3t7GxQUBAgMHqLq9PREVFwc3NTbc3AgCjR49GvXr18P333xdrz8bGBq+//jquX79ukPru3bsHpVIJW1tbAIXrz8nJqdTpleHo6KjbQ1WpVHByckJycrLeNf/55584cOAAPvzwQ1hbWwMAlEolhg0bhlq1amHx4sXw9fWFr68vFi9ejPz8/HLbzMjIwPHjx7Fs2TKcPXsWqampetcJlL5+y5KSkoL69etDpVIBKFx3zZs3r9DysrOzUa9ePf2K1pNJBYibmxsSEhJw//597Ny5EwMHDiz2malTp2Lt2rV49OhRFVT4PwkJCXj55Zd1neWx9u3bIyEhQfc6JCQEO3bswN27d592iRVy+vRp1K9fv9zd4KtXr8LPz0/373FIGkNF+gFQOFx0+PBh3ZDm3yUmJuLatWto0aIFLl68iLZt2xqtZqD8PpGQkIB27doVm69du3ZF+sxjKSkpOHPmTKX/mJemZcuWaNu2Lby9vTF58mRs3rwZGRkZpU6XlZaWhv3798Pb21vvmn///Xe88sorJf6h3LZtGy5duoSYmBjExMTg999/x7Zt28pt8/vvv4e3tzcaNGiAHj16YOfOnXrXCZS+fsvSt29fXL16FT179sSsWbMQFxeHvLy8Uj8fGxsLPz8/eHt7IyoqChMnTjRI7bJMKkDMzMzQp08fxMfHIz4+Hr6+vsU+06JFC7i6uiIqKqoKKvwfUcY9KJ8c923QoAGGDh2KNWvWPI2yKmzy5Mno1asXAgMDERoaqtsCKs2rr76KuLg43T9jdtzy+sHRo0fh5+eHzp07Iz09vcgWPQB8+OGH8PPzw+zZs7Fw4UI0a9bMaLU+qbw+UdE+s379evj5+SEkJATjxo2Dh4eHQepTKBRYs2YNoqKi4O7ujkOHDmHAgAF48OBBidMzMzMrvYzs7GyEhIRgzJgxBgu+0hw7dgz+/v6643mDBg3CsWPHyp0vJiYG/v7+AIru7eqrtPVb1np0cHBAfHw8Pv74YzRp0gSRkZEYP358qZ8fOHAg4uLicPDgQUyfPh1Tp041SO2yTCpAgMIvNCIiAo6Ojqhfv36JnwkNDcXmzZuRlZX1lKv7H0dHR9y8ebNY5zh37hycnZ2LTBs3bhwOHDiAmzdvPs0SyxQREYH9+/dj+fLlmDVrFu7du1fVJRVRVj/w8PBAXFwcDh06BDMzM6xcubLI+3PnzkVcXBy2b9+uC5/WrVvj/PnzRq25vD7RokUL/Prrr8Xm++2334r0maCgIMTFxSEmJkZ3AoCh6xwxYgQ2bdoEGxsbnDx5sszpJVmwYIFub/TatWsAgNzcXAQHB8PT0xNjxowxSK1OTk64ceMG7t+/b5D2Lly4gMTERMyZMwc+Pj6YMWMGkpKScPr0aYO0D1RuPQKFQ3IuLi4YP348oqKicOTIEWRmZpa4jp/Uu3dvXLp0Cenp6QarvbJMLkBeeuklTJ06FRMmTCjzM7169cJXX331FCsrqnHjxujduzc++OADqNVqAIVDGF999RWmTJlS5LM2NjZ45513sHbt2qootUx9+vSBp6cn1q1bV9WlFFGRfmBtbY0FCxZgy5Yt5Q4R9uvXD+np6diwYYNuTyA7O7tCQx4VVV6fCAwMxIkTJ7B3717dPJs3b4alpSW6d+9usDpKk5KSgrNnz+pe37lzB+np6bCzsytxeuPGjUtta/78+bq90WbNmkGtViM4OBjt2rVDaGiowWpu0qQJfHx8EB4ejuzsbABAfn4+vvvuO7i5uSE2NhZarRZarRaxsbHl7q1FR0dj3Lhx+Omnn3T/Jk2ahOjoaL1rLW39lrUeL1y4gKSkJN3rixcvol69eqhbt26xdfx3J06cgK2tbbEh06fJZE7jfdKwYcPK/cyECROKjV0+ePAAXbp00b1u1qwZNm/ebOjydObPn4/ly5ejb9++MDMzQ0pKCrZv345WrVoVO504MDAQX3/9dbE2evfurRu+sLKywv79+41Wb2nCwsIwaNAgvPvuu6V+5vExkMccHByMfnJARfpBy5YtdScqzJkzp9TP1a5dG1FRUVi6dCnefPNN1KlTB0qlEm+99ZYhSy6zTwDAl19+iSVLlmDZsmXIyspC8+bN8eWXX0KpNP6vYl5eHlatWoXbt2+jVq1aKCgowJQpU/DCCy9g3rx5xaZXZghqx44dOHnyJDIzM3HkyBEAhX07JCRE77o/+eQTrF69GoMHD4aFhQUKCgrQtWtXTJ06Fbdv39YNR3l5eRU5JXvr1q2609MBYOzYsYiPj8eWLVuKtO/r64sBAwZg7ty5qF27tnSdpa3fstZjRkYGFixYgOzsbKhUKlhZWWH16tW6Ez3+LjY2FkePHoUQAkqlEitXriz1s08DHyhlIFqtFuHh4bhz5w4iIyN1Z+FQzVVen7h69SomTJiAsWPHGvRaFKKnhQFCRERSTO4YCBERVQ8MECIiksIAISIiKQwQIiKSwgAhIiIpDBAiIpLCACEiIikMECIiklLm/RMSEq5Aq9U+rVqIiMjEWFhYwNHxtRLfKzNAtFotGjV6xShFERGR6UtOvlHqexzCIiIiKQwQIiKSUul7SNepbQmVpeFvPa1R5+FhjrrU96dMmYguXbwxaNA/dNOEEBg8eADmzv0AHTp0rPCytm79Bj179sFzzz0HAIiJ2QG1+hGGDw+Uqv2LLyKRm5uLyZP1fzpYcnIyhgzxQ7Nmr6KgoAB5eXlo184Z48YFwcHheQDARx8tRL9+vmjfvoPeyzOU3bu/xy+//IyPP15qtGXYWCmhrGX4uxznPVIjK7f0x4gChut/ptz3nhW1rS1gaVH2EzZlqLUa5GSXf0x44MB+sLS0hIWFBR49ykXTpq9i5MjRaNu2nd7ftzEkJyfjnXcCsX//T5Wet9JJoLJUYmHY7kovqDzhy3zLDJD+/f3w7bdRRX6Bz5w5BYXCDM7OFftDWlBQADMzM2zd+i1cXd11v8RPtmlMISHvYt68BWjUqFGZn7O2tkFU1FYAhcehNm3agHfffQfffLMN1tY2mDMn/GmUa3KUtSzxi99gg7frGRcNlBMg+va/qu57NYmlhQpDt+n/HJK/2z5sLXJQsZOKFi1agldfbQ4A+M9//o1p0ybhs89WP3Pft0k+UKokXbp4Y8mSj3H9+jU0bVr4dK7du79Hv34DYGZmhq+/3oyDB/+NvLx82NvbY/bsebCza4AvvojE9evXkJ2djZSUO+jduy/u3UvF7NnvQaVSYeHCRThw4IciW3FfffUlfvhhH8zMzGBlZYV1675ERkY65s2bjYcPs6HRaODh4YVJk6aUVbJBWFhYICgoBCdPHse+fXvwj38MQ0jIuxgxYiS8vLpg//692LZtC/LyCjv2pElT4OrqDgA4d+4Mli79BADQsaMrDh/+D5Yti8CrrzbH779fxPLlS5CbmwsrKytMm/YenJxa67ZGBg4chGPHfsGjR48we3Y42rd3Rl5eHsLCJuP+/ftQq9VwcmqNmTPnwsLCwujroaqV1f+ior56JvseGUa3bm/i998v4ttvo9C0aTPd952YeAVLl36M3NxH0GjUGDhwEAICRgAA7t69i4UL5yEtLQ0vvtgYQgh06tQZQ4YEIC0tDUuWLEJS0i0AwIgRo9C3b+GjmwcO7Ie+fX1x8uRx3Lt3DyNGjMSQIQEAgIiIFTh79jS0Wi1sbW0xZ858NGxY9sZseapNgFhYWKBXrz7Yvft7TJo0BQ8fPsShQwexdesO7N0bj9u3b2HDhq+gUCgQHf0dVq5cgYULPwIAXLx4AV999Q1sbQufrR0Xt7PIFsKT4uN34eefD2H9+k2oU6cO7t/PhEKhgLW1DT799DPUrl0beXlahIb+E8eO/YLOnT2fys/v5NSmxOcid+rUGT17Fj7V8MaNPzFxwrEHbgAADLZJREFUYjB27doHjUaDefNm4//9v0Vo374DDh78Cdu3Fz6JTavVYtasGZg7dz5cXd1x8uQJzJo1Azt2xAEA7t/PxOuvt0VIyETs27cHq1dH4IsvNsHc3BwLFy5CvXq2EEJg4cJw7NoV98xtVZWktP43Y8ZMnDt35pnue6S/1q3b4OefD+k2PgCgYcNGWLUqEiqVCjk5ORgzZiTc3TujadNmWL58CTp0cMWYMePw11/JGDFiGDp16gwAWL58CZo1exWLFy/DvXupGD16BFq0aKnrU48ePcKGDV8hOTkZI0YMQb9+A1C7dm2MGjVat6ESF7cTq1dH4MMPP9Hr56o2AQIUDiNMmTIREyZMwoEDP6Bt23ZwcHgeR44cxqVLv+PttwsfT5qfn486dax183l4eOp+gcvzyy8/Y9CgIahTpw4AoF69wucNFxQUYNWqz3D+/K8QQiA9PQ1XriSU+0u8ePFHuHDhPAAgKekWpk2bpNtiX7lyjW4oozylPfcrKSkJ69bNRmrqXSiVSqSnpyEt7R7S09NhaWmpO07i7e0DGxsbAMCNG3/CwkKp21Nxc3OHhYUSN278idq166B27drw8ip8NHCbNq8jImKFbh18800Ujh37BQUFBXjw4AFq1apVofqfBSX1P1Pue2Q6Svr9ffToEZYsWYTExCswMzPDvXupSEy8gqZNm+HMmVOYNu09AIVB4+rqppvv//7vJEJDpwEAGjSwR+fOXjh9+pQuQHr06AUAaNSoEWxs6uLu3RQ0adIUx479gh07tiM3Nxf5+fkG+bmqVYC89pojGjRogGPHfsHu3d8jIKDwl1YIgXfeGYv+/QeWOJ+Vlfxzjh/bsuVfyMp6gI0bv4alpSU+/vj/Qa0u/ZjNY++//7/ndFf0GEhJLl26iN69+xWbHh4+G5MnT0XXrt1QUFAAb28PqNWaSrf/JIsnDkAqFOa6zvbDD3vx669nERm5EXXq1MHmzRtx8+ZNvZZVnZTU//797x9Ntu+R6bh06fdie52RkZ/Dzq4B5s1bAKVSicmTJxjke1Wpnvz9VSA/Px9//ZWMzz5bjk2botCo0Yv47bdfER4+W+9lVbvTePv398OGDetw69YNdOniDQB4440uiI7+Dg8ePAAAaDQaXLmSUGobderUQXZ2donveXq+gZiY7/Dw4UMAhcM5AJCVlYUGDRrA0tISd+/exeHDhwz4U5VOq9Viw4b/3969B0VZ73Ecf8MugogOcLycgACvM2VnkquQNc6xyVHRJMx7eLwdJzEFRbFiTh5NKLwgKikOjXeUTokiqFmdKMLwAqLNCTsFZl6YNJmYli32BuePbZ8jSuCui7jyff3H/va57Tw8v+f3e37P57eNGzduMGrU6DvKNRoNPj6+ABQU5KPXmyuPgIBAdLoGzp8/B0Bx8WdoNBqlzGAwUl5+BoCystMYjUYCAgJb3ReNph5PT6/ffz8NH330ob0O02Hcfv49zOeesI/i4s/Iy3v/jpFXGo2GPn36oFarqa6u4vz5CqUsKCiEo0cLALh+/UfKys4oZWFh4eTn5wFQW3uT0tISQkPDWt0HrVaLi4sab+8/0djYyMGDH9jl2ByqBQIwcuRoNm/OYPz4GKUraPTosdTV1TF//lzA3CKJiZnIwIGDWlzHpElTWb36n7i5ubFqVWqzsjFjxvLTTzeYO/dvqNVqunZ1JyvrXSZNmkJy8nKmTZtI7969mzUp7a2+XkNs7BRMJpMyjDc7ewceHt3v+O7ixYkkJS2he/ceREZGKt0eXbp0YeXKVNasMR9fUFAIXl7eeHh44OLiwltvrW32ED01dW2bD8PHjImiuPgzJk+OwcvLiyefDOp0d8K3n38P27kn7OP115OUYbyBgf1IT9/ME0/8hdLSE8p3Zs2ay8qV/+Dw4UP4+wcwZEiQUrZkyTJWrnyD48eP4ePjw+OPD8bDw+P3siTS0lKYPn0SAHFxi+jXr3+r+zNgwEBGjHiOqVNfxNPTk6eeGkZFxdl7Pk6npj/qXAe+/rryjiiTjnoPRFhPq9Uq/enl5Wd4880V5OUV4uzscA1PoGPfAxGOo6PfA7GHhoYG1Go1arWamzd/YtasWDIzs9rsJWgPNTU/MHjw4y2WWV0TaH/VyYXeQRQV/Zvc3ByampqUFomjVh6A+SIvF3rRhl/rDXf9vsaD6sqVy6xa9QZNTU0YjUbmzJnXIZVHW6xugQghhOg8WmuBOO7tqBBCiA4lFYgQQgibSAUihBDCJlKBCCGEsInDVSDR0VFMnhxDbOwUJk+OITX1TYxGA9nZWUrkhi3udXnx8IuOjqK6uqrZZzNnTqe8vKzV5QoLD3P58h/P6iaEo7J6GG+PbmpUXew/Ft+k1/GL9u6GaFrC6EwmEy+/PIeiIutz7IXj6eruils7vIPUoDPyWzsOTT9ypABPT0/8/a0b0WiJgHdycmqnPRPi3lj936jq4srFFPvPydAv+QDcZQViodfr0en09OjRo9nnJpOJd97ZxMmTXwIQEfEUCxYsQqVSUV+vISNjPRcuVOLk5MSQIUEsXfpqs+Wrqr5jxYpkEhOXWzVRlWhfbq5qxiXm2329BevH33MFotXWk5GRTnX1d+h0OkJCwoiPX8KxY4V8800l6elr2bZtCwsXLiY8fOhdTz+Qnb2TkpJicnJ2A+Dn9yjLlyfj7e2NwWBg3bq3OXu2HC8vbwYNGkRtba0yqVdr27h8+Qfq6+upqbmGr68fqalpuLl1xWAwkJWVSUXFWfR6PQMGDCQp6XXc3d1bjRGPiAjm009LcHd3b/a3s7MTq1at4OLFatRqNQEBgaSkpN3Tby0eHA4XZQIo8ylcu3aV8PAIhg6N5Kuvzivlhw7l8e23/2XXrn0ALF78CocO5TFhwkQ2bFhH167u7NmTi7OzM3V1Pzdb9+nTp9i4cT2rV7/dLHpZCPj/uWdhCZPMyEgnODiY5OQ3aGxsZMWKZAoK8omOjuHIkUJl/hbAqukHqqur2LJlEzt35tCzZy+2bdvC+vVppKSkcfDgAa5f/5H9+z/AZDIRF/d3ZdbKtrZx4UIlO3bsxcPDg/j4BXz44TGio2PYs2cX3bp1Z/v2PQBkZm5k167tzJ//Spsx4i05ebIUrbae3NwDAEpmmHg4OGQFYunC0ul0vPbaMnJzc5qVnzlziqiocUq2U1TU83z+eRETJkzkxIkv2LkzR3kj+9ao7VOnSjl58ks2btxCr1697t8BCYdx+1weM2eaJwAqKfmcysr/sG/fXsAcRdG7d+8W12FNBHx5eRmRkU/Ts6f5fHzhhQm89NKU38vOMGpUlBJ5MXLkKM6dq7irbURERCrx/oMHP8G1a1eV49BqtRQVfQKYW/mWXK+2YsRbMnDgIC5dusTatW8RHBzKsGFPt/bzCgfjkBWIhaurK8OGPcOJE1/w2GMtvylpDX//AC5erObChUp69Rpuhz0UnUVTUxNr1qTj6+t3V99t7wj4trbR5ZbnmJbIb8tyy5a9SmiodYGNKpWKpqZGgGYBm76+fuzb9z5lZacpLT3B1q2Z5OT8C1dX+z9HFfefw43CulVjYyMVFeX4+/s3+zwsbChHjxZiNBowGg0cPVpIeHgEYI7M3rt3tzLBy61dWI884sOmTVvYunUzH398/P4diHB4zzwznN27dygX4rq6n6mpuQbcGeFuTQR8SEgopaUl1NbeBCA/P4/wcPNEYMHBoRw/fgyj0YhOp+OTTz6yaRu3H8f+/XtpaGgAzIGc339vngmztRhxP79Hqaz8GjDPG2Nx48Z1VCpnhg//KwkJidTV/SzdWA8Rh2yBWPqhDQYD/fsPYPbsebz33j6lPDo6hqtXrzBjhrn5PnRoJOPHvwBAQsJSNmxYx7RpE1GpVAQFhZCYmKQs26fPn8nMzCI+fgE6nY6xY5+/vwcnHFJCwlIyMzcSGzsFJycnXFxcSEhYio+PL9HRMWzatIGcnN0sXLjYqgj4/v0HEBe3iEWL4gDzHb1lkrKYmBepqvpWiegODOyrLGdtzLzFjBkzyc7exuzZscoIsDlz5tG3b79WY8Tj45eQlpZCt24ePPvsSGV9VVXmZzhgvuGbMWO2dA8/RKwOU3wQhvGKzslRh/G2J0tkv16vZ9myBEaMeE65WRLCHuwa5/6L1mj1cFsh7OG3X3UOe6FvLwsXzsdg0KPX6wkLCycqalxH75LoRByyC0sIYbZ9++6O3gXRiTn0Q3QhhBAdRyoQIYQQNpEKRAghhE2kAhFCCGGTVh+iu7i4UFMjMdRCCNFZWSKhWtLqeyBCCCHEH5EuLCGEEDaRCkQIIYRNpAIRQghhE6lAhBBC2EQqECGEEDb5HySSnQCjCLj8AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x576 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import itertools\n",
    "sns.set()\n",
    "def flip(items, ncol):\n",
    "    return list(itertools.chain(*[items[i::ncol] for i in range(ncol)]))\n",
    "\n",
    "\n",
    "tasks = ['MNLI', 'QNLI', 'RTE', 'MRPC', 'QQP', 'SST-2', 'CoLA', 'STS-B']\n",
    "\n",
    "heterogeneous = np.array([patterns[task]['attention']['other'] * 100 for task in tasks])\n",
    "block =  np.array([patterns[task]['attention']['block'] * 100 for task in tasks])\n",
    "diagonal = np.array([patterns[task]['attention']['diagonal'] * 100 for task in tasks])\n",
    "vertical = np.array([patterns[task]['attention']['vertical'] * 100 for task in tasks])\n",
    "vertical_diagonal = np.array([patterns[task]['attention']['mix'] * 100 for task in tasks])\n",
    "\n",
    "wn_hetero = np.array([patterns[task]['weight normed']['Heterogeneous'] * 100 for task in tasks])\n",
    "wn_block =  np.array([patterns[task]['weight normed']['Block'] * 100 for task in tasks])\n",
    "wn_diagonal = np.array([patterns[task]['weight normed']['Diagonal'] * 100 for task in tasks])\n",
    "wn_vertical = np.array([patterns[task]['weight normed']['Vertical'] * 100 for task in tasks])\n",
    "wn_vertical_diagonal = np.array([patterns[task]['weight normed']['Vertical + Diagonal'] * 100 for task in tasks])\n",
    "N = len(heterogeneous)\n",
    "\n",
    "ind = np.arange(len(tasks))    # the x locations for the groups\n",
    "width = 0.35       # the width of the bars: can also be len(x) sequence\n",
    "\n",
    "\n",
    "fig, (ax1, ax2) = plt.subplots(2, figsize=(6, 8))\n",
    "ax1.set_title(\"Attention Patterns\")\n",
    "stuff_to_stack = [heterogeneous, block, diagonal, vertical, vertical_diagonal]\n",
    "ps_1s = []\n",
    "summed = None\n",
    "for i, stuff in enumerate(stuff_to_stack):\n",
    "    p = ax1.bar(ind, stuff, width, bottom=summed)\n",
    "    ps_1s.append(p)\n",
    "    if summed is None:\n",
    "        summed = stuff\n",
    "    else:\n",
    "        summed += stuff\n",
    "ax1.set_xticks(ind)\n",
    "ax1.set_xticklabels(tasks)\n",
    "ax1.set_yticklabels([], [])\n",
    "\n",
    "ax2.set_title(\"Weight Normed Attention Patterns\")\n",
    "stuff_to_stack_2 = [wn_hetero, wn_block, wn_diagonal, wn_vertical, wn_vertical_diagonal]\n",
    "ps_2s = []\n",
    "summed = None\n",
    "for i, stuff in enumerate(stuff_to_stack_2):\n",
    "    p = ax2.bar(ind, stuff, width, bottom=summed)\n",
    "    ps_2s.append(p)\n",
    "    if summed is None:\n",
    "        summed = stuff\n",
    "    else:\n",
    "        summed += stuff\n",
    "ax2.set_xticks(ind)\n",
    "ax2.set_xticklabels(tasks)\n",
    "\n",
    "ax2.set_yticklabels([], [])\n",
    "#fig.subplots_adjust(bottom=0.3, wspace=0.33)\n",
    "ax2.legend(flip(list(reversed(ps_2s)), 3), flip(list(reversed([\"Heterogeneous\", \"Block\", \"Diagonal\", \"Vertical\", \"Vertical + Diagonal\"])), 3), loc='upper center', bbox_to_anchor=(0.5, -0.1),fancybox=False, shadow=False, ncol=3)\n",
    "\n",
    "fig.tight_layout()\n",
    "fig.savefig(\"common_components/patterns.pdf\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Heterogeneous', 'Vertical', 'Block', 'Vertical + Diagonal', 'Diagonal']"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " flip([\"Heterogeneous\", \"Block\", \"Diagonal\", \"Vertical\", \"Vertical + Diagonal\"], 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}