experiment_analysis/statistics_on_pruning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pathlib\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import json\n",
    "import itertools\n",
    "from statsmodels.stats.contingency_tables import cochrans_q\n",
    "from statsmodels.stats.inter_rater import fleiss_kappa\n",
    "import krippendorff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_head_data(experiments_path):\n",
    "    head_data = {}\n",
    "    for task_dir in experiments_path.iterdir():\n",
    "        head_data[task_dir.stem] = {}\n",
    "        for seed_dir in task_dir.iterdir():\n",
    "            head_mask = np.load(seed_dir / \"head_mask.npy\")\n",
    "            head_data[task_dir.stem][seed_dir.stem] = {\n",
    "                \"head_mask\": head_mask,\n",
    "            }\n",
    "    return head_data\n",
    "def load_mlp_data(experiments_path):\n",
    "    mlp_data = {}\n",
    "    for task_dir in experiments_path.iterdir():\n",
    "        mlp_data[task_dir.stem] = {}\n",
    "        for seed_dir in task_dir.iterdir():\n",
    "            mlp_mask = np.load(seed_dir / \"mlp_mask.npy\")\n",
    "            mlp_importance = np.load(seed_dir / \"mlp_importance.npy\")\n",
    "            mlp_data[task_dir.stem][seed_dir.stem] = {\n",
    "                \"mlp_mask\": mlp_mask,\n",
    "                \"mlp_importance\": mlp_importance\n",
    "            }\n",
    "    return mlp_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiments_path = pathlib.Path(\"../masks/heads_mlps\")\n",
    "heads = load_head_data(experiments_path)\n",
    "\n",
    "experiments_path = pathlib.Path(\"../masks/heads_mlps_hans\")\n",
    "hans_heads = load_head_data(experiments_path)\n",
    "\n",
    "for k, v in hans_heads.items():\n",
    "    heads[k] = v\n",
    "\n",
    "experiments_path = pathlib.Path(\"../masks/heads_mlps\")\n",
    "mlps = load_mlp_data(experiments_path)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "experiments_path = pathlib.Path(\"../masks/heads_mlps_hans\")\n",
    "hans_mlps = load_mlp_data(experiments_path)\n",
    "\n",
    "for k, v in hans_mlps.items():\n",
    "    mlps[k] = v\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def cochrans_q_masks(masks):\n",
    "    inp = np.array(masks).transpose()\n",
    "    return cochrans_q(inp)\n",
    "\n",
    "\n",
    "\n",
    "def krippendorff_alpha_tasks_separate(data, mask=\"head_mask\"):\n",
    "    for task in sorted(data.keys()):\n",
    "        krippendorff_alpha_tasks(data, [task], mask)     \n",
    "        \n",
    "def krippendorff_alpha_tasks(data, tasks, mask=\"head_mask\"):\n",
    "    masks = []\n",
    "    seeds = sorted(data[tasks[0]].keys())\n",
    "    for task in tasks:\n",
    "        for seed in seeds:\n",
    "            masks.append(data[task][seed][mask].reshape(-1))\n",
    "    alpha = krippendorff.alpha(masks)\n",
    "    print(','.join(tasks))\n",
    "    print(\"---------\")\n",
    "    print(f'alpha: {alpha}')\n",
    "\n",
    "    \n",
    "def fleiss_kappa_tasks_separate(data, mask=\"head_mask\"):\n",
    "    for task in sorted(data.keys()):\n",
    "        fleiss_kappa_tasks(data, [task], mask)     \n",
    "        \n",
    "def fleiss_kappa_tasks(data, tasks, mask=\"head_mask\"):\n",
    "    masks = []\n",
    "    \n",
    "    good_mask_sum = np.zeros_like(data[\"MNLI\"][\"seed_42\"][mask].reshape(-1))\n",
    "    bad_mask_sum =  np.zeros_like(data[\"MNLI\"][\"seed_42\"][mask].reshape(-1))\n",
    "    seeds = sorted(data[tasks[0]].keys())\n",
    "    for task in tasks:\n",
    "        for seed in seeds:\n",
    "            good_mask_sum = good_mask_sum + data[task][seed][mask].reshape(-1)\n",
    "            bad_mask_sum = bad_mask_sum + (1 - data[task][seed][mask].reshape(-1))\n",
    "    table = np.array([good_mask_sum, bad_mask_sum]).transpose()\n",
    "    kappa = fleiss_kappa(table)\n",
    "    print(','.join(tasks))\n",
    "    print(\"---------\")\n",
    "    print(f'Feliss Kappa: {kappa}')    \n",
    "\n",
    "def print_p_value_tasks_separate(data, mask=\"head_mask\"):\n",
    "    for task in sorted(data.keys()):\n",
    "        print_p_value_tasks(data, [task], mask)        \n",
    "        \n",
    "def print_p_value_tasks(data, tasks, mask=\"head_mask\"):\n",
    "    masks = []\n",
    "    seeds = sorted(data[tasks[0]].keys())\n",
    "    for task in tasks:\n",
    "        for seed in seeds:\n",
    "            masks.append(data[task][seed][mask].reshape(-1))\n",
    "    test_result = cochrans_q_masks(masks)\n",
    "    print(','.join(tasks))\n",
    "    print(\"---------------------------------------------------------------------\")\n",
    "    print(f'p-value: {test_result.pvalue}')\n",
    "    print(f\"{'Null hypothesis (all seeds are similar) is rejected.' if test_result.pvalue < 0.05 else 'Null hypothesis (all seeds are similar) is not rejected.'}\")\n",
    "    \n",
    "    if len(tasks) == 1:\n",
    "        masks_combos = list(itertools.combinations(range(len(masks)), 2))\n",
    "    else:\n",
    "        masks_combos = []\n",
    "        for i in range(len(seeds)):\n",
    "            for j in range(len(seeds)):\n",
    "                if i < j:\n",
    "                    mask_1_idx = i\n",
    "                    mask_2_idx = len(seeds)  +  j\n",
    "                    masks_combos.append((mask_1_idx, mask_2_idx))\n",
    "    similar_masks_combos = []\n",
    "    for mask_1, mask_2 in masks_combos:\n",
    "        r = cochrans_q_masks([masks[mask_1], masks[mask_2]])\n",
    "        if r.pvalue >= 0.05:\n",
    "            task1_name, seed1_name = tasks[mask_1 // len(seeds)], seeds[mask_1 % len(seeds)]\n",
    "            task2_name, seed2_name = tasks[mask_2 // len(seeds)], seeds[mask_2 % len(seeds)]\n",
    "            similar_masks_combos.append((f\"{task1_name}-{seed1_name}\", f\"{task2_name}-{seed2_name}\"))\n",
    "    \n",
    "    print(f\"Total mask pairs where Null hypothesis is not rejected - {len(similar_masks_combos)}\")\n",
    "    print(f\"Total mask pairs - {len(masks_combos)}\")\n",
    "    print(f\"Percentage - {len(similar_masks_combos)/ len(masks_combos)}\")\n",
    "    print(\"\\nSimilar Mask Pairs:\\n\")\n",
    "    print(\"\\t\".join([\",\".join(p) for p in similar_masks_combos]))\n",
    "    print(\"\\n\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Seeds in a Task"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Heads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CoLA\n",
      "---------------------------------------------------------------------\n",
      "p-value: 2.113894110905964e-11\n",
      "Null hypothesis (all seeds are similar) is rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 3\n",
      "Total mask pairs - 10\n",
      "Percentage - 0.3\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "CoLA-seed_1337,CoLA-seed_42\tCoLA-seed_1337,CoLA-seed_71\tCoLA-seed_166,CoLA-seed_42\n",
      "\n",
      "\n",
      "\n",
      "HANS\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.004749695087342902\n",
      "Null hypothesis (all seeds are similar) is rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 7\n",
      "Total mask pairs - 10\n",
      "Percentage - 0.7\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "HANS-seed_1337,HANS-seed_42\tHANS-seed_1337,HANS-seed_71\tHANS-seed_1337,HANS-seed_86\tHANS-seed_166,HANS-seed_86\tHANS-seed_42,HANS-seed_71\tHANS-seed_42,HANS-seed_86\tHANS-seed_71,HANS-seed_86\n",
      "\n",
      "\n",
      "\n",
      "HANS_MNLI\n",
      "---------------------------------------------------------------------\n",
      "p-value: 1.0701419296602784e-05\n",
      "Null hypothesis (all seeds are similar) is rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 4\n",
      "Total mask pairs - 10\n",
      "Percentage - 0.4\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "HANS_MNLI-seed_1337,HANS_MNLI-seed_42\tHANS_MNLI-seed_1337,HANS_MNLI-seed_86\tHANS_MNLI-seed_166,HANS_MNLI-seed_42\tHANS_MNLI-seed_71,HANS_MNLI-seed_86\n",
      "\n",
      "\n",
      "\n",
      "MNLI\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.39484537126862373\n",
      "Null hypothesis (all seeds are similar) is not rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 9\n",
      "Total mask pairs - 10\n",
      "Percentage - 0.9\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "MNLI-seed_1337,MNLI-seed_166\tMNLI-seed_1337,MNLI-seed_42\tMNLI-seed_1337,MNLI-seed_71\tMNLI-seed_1337,MNLI-seed_86\tMNLI-seed_166,MNLI-seed_42\tMNLI-seed_166,MNLI-seed_71\tMNLI-seed_166,MNLI-seed_86\tMNLI-seed_42,MNLI-seed_86\tMNLI-seed_71,MNLI-seed_86\n",
      "\n",
      "\n",
      "\n",
      "MNLI_TWO\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.04886971480018394\n",
      "Null hypothesis (all seeds are similar) is rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 8\n",
      "Total mask pairs - 10\n",
      "Percentage - 0.8\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "MNLI_TWO-seed_1337,MNLI_TWO-seed_42\tMNLI_TWO-seed_1337,MNLI_TWO-seed_71\tMNLI_TWO-seed_1337,MNLI_TWO-seed_86\tMNLI_TWO-seed_166,MNLI_TWO-seed_42\tMNLI_TWO-seed_166,MNLI_TWO-seed_86\tMNLI_TWO-seed_42,MNLI_TWO-seed_71\tMNLI_TWO-seed_42,MNLI_TWO-seed_86\tMNLI_TWO-seed_71,MNLI_TWO-seed_86\n",
      "\n",
      "\n",
      "\n",
      "MNLI_TWO_HALF\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.10751157701176865\n",
      "Null hypothesis (all seeds are similar) is not rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 8\n",
      "Total mask pairs - 10\n",
      "Percentage - 0.8\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "MNLI_TWO_HALF-seed_1337,MNLI_TWO_HALF-seed_166\tMNLI_TWO_HALF-seed_1337,MNLI_TWO_HALF-seed_42\tMNLI_TWO_HALF-seed_1337,MNLI_TWO_HALF-seed_71\tMNLI_TWO_HALF-seed_1337,MNLI_TWO_HALF-seed_86\tMNLI_TWO_HALF-seed_166,MNLI_TWO_HALF-seed_42\tMNLI_TWO_HALF-seed_166,MNLI_TWO_HALF-seed_71\tMNLI_TWO_HALF-seed_166,MNLI_TWO_HALF-seed_86\tMNLI_TWO_HALF-seed_71,MNLI_TWO_HALF-seed_86\n",
      "\n",
      "\n",
      "\n",
      "MRPC\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.013837912255726509\n",
      "Null hypothesis (all seeds are similar) is rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 7\n",
      "Total mask pairs - 10\n",
      "Percentage - 0.7\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "MRPC-seed_1337,MRPC-seed_166\tMRPC-seed_1337,MRPC-seed_42\tMRPC-seed_1337,MRPC-seed_71\tMRPC-seed_1337,MRPC-seed_86\tMRPC-seed_166,MRPC-seed_86\tMRPC-seed_42,MRPC-seed_71\tMRPC-seed_71,MRPC-seed_86\n",
      "\n",
      "\n",
      "\n",
      "QNLI\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.0329959709332481\n",
      "Null hypothesis (all seeds are similar) is rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 6\n",
      "Total mask pairs - 10\n",
      "Percentage - 0.6\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "QNLI-seed_1337,QNLI-seed_166\tQNLI-seed_1337,QNLI-seed_71\tQNLI-seed_1337,QNLI-seed_86\tQNLI-seed_166,QNLI-seed_42\tQNLI-seed_166,QNLI-seed_86\tQNLI-seed_42,QNLI-seed_86\n",
      "\n",
      "\n",
      "\n",
      "QQP\n",
      "---------------------------------------------------------------------\n",
      "p-value: 2.970583205868185e-07\n",
      "Null hypothesis (all seeds are similar) is rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 4\n",
      "Total mask pairs - 10\n",
      "Percentage - 0.4\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "QQP-seed_1337,QQP-seed_166\tQQP-seed_1337,QQP-seed_42\tQQP-seed_166,QQP-seed_42\tQQP-seed_71,QQP-seed_86\n",
      "\n",
      "\n",
      "\n",
      "RTE\n",
      "---------------------------------------------------------------------\n",
      "p-value: 2.6922221869717934e-36\n",
      "Null hypothesis (all seeds are similar) is rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 2\n",
      "Total mask pairs - 10\n",
      "Percentage - 0.2\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "RTE-seed_166,RTE-seed_71\tRTE-seed_71,RTE-seed_86\n",
      "\n",
      "\n",
      "\n",
      "SST-2\n",
      "---------------------------------------------------------------------\n",
      "p-value: 3.417832305517236e-08\n",
      "Null hypothesis (all seeds are similar) is rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 2\n",
      "Total mask pairs - 10\n",
      "Percentage - 0.2\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "SST-2-seed_42,SST-2-seed_86\tSST-2-seed_71,SST-2-seed_86\n",
      "\n",
      "\n",
      "\n",
      "STS-B\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.05786884035446416\n",
      "Null hypothesis (all seeds are similar) is not rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 9\n",
      "Total mask pairs - 10\n",
      "Percentage - 0.9\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "STS-B-seed_1337,STS-B-seed_166\tSTS-B-seed_1337,STS-B-seed_42\tSTS-B-seed_1337,STS-B-seed_71\tSTS-B-seed_1337,STS-B-seed_86\tSTS-B-seed_166,STS-B-seed_42\tSTS-B-seed_166,STS-B-seed_86\tSTS-B-seed_42,STS-B-seed_71\tSTS-B-seed_42,STS-B-seed_86\tSTS-B-seed_71,STS-B-seed_86\n",
      "\n",
      "\n",
      "\n",
      "WNLI\n",
      "---------------------------------------------------------------------\n",
      "p-value: 2.42151439227382e-123\n",
      "Null hypothesis (all seeds are similar) is rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 0\n",
      "Total mask pairs - 10\n",
      "Percentage - 0.0\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sai/miniconda3/envs/bertexp/lib/python3.7/site-packages/statsmodels/stats/contingency_tables.py:1411: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  / (k * count_row_ss - np.sum(count_row_success**2)))\n"
     ]
    }
   ],
   "source": [
    "print_p_value_tasks_separate(heads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CoLA\n",
      "---------\n",
      "alpha: 0.22920627475942013\n",
      "HANS\n",
      "---------\n",
      "alpha: 0.1511915220733583\n",
      "HANS_MNLI\n",
      "---------\n",
      "alpha: 0.28187521458222786\n",
      "MNLI\n",
      "---------\n",
      "alpha: 0.25860365198711066\n",
      "MNLI_TWO\n",
      "---------\n",
      "alpha: 0.2586712903250721\n",
      "MNLI_TWO_HALF\n",
      "---------\n",
      "alpha: 0.27296884185773074\n",
      "MRPC\n",
      "---------\n",
      "alpha: 0.21637124992829693\n",
      "QNLI\n",
      "---------\n",
      "alpha: 0.30249353262431733\n",
      "QQP\n",
      "---------\n",
      "alpha: 0.1923483546980309\n",
      "RTE\n",
      "---------\n",
      "alpha: 0.1810617760617761\n",
      "SST-2\n",
      "---------\n",
      "alpha: 0.32545842217484\n",
      "STS-B\n",
      "---------\n",
      "alpha: 0.2226213727678572\n",
      "WNLI\n",
      "---------\n",
      "alpha: -0.24826388888888884\n"
     ]
    }
   ],
   "source": [
    "krippendorff_alpha_tasks_separate(heads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CoLA\n",
      "---------\n",
      "Feliss Kappa: 0.22813420962464326\n",
      "HANS\n",
      "---------\n",
      "Feliss Kappa: 0.15001104022155096\n",
      "HANS_MNLI\n",
      "---------\n",
      "Feliss Kappa: 0.28087644551473623\n",
      "MNLI\n",
      "---------\n",
      "Feliss Kappa: 0.2575725895259203\n",
      "MNLI_TWO\n",
      "---------\n",
      "Feliss Kappa: 0.257640223291972\n",
      "MNLI_TWO_HALF\n",
      "---------\n",
      "Feliss Kappa: 0.2719576396639385\n",
      "MRPC\n",
      "---------\n",
      "Feliss Kappa: 0.21528139476292904\n",
      "QNLI\n",
      "---------\n",
      "Feliss Kappa: 0.30152363649730873\n",
      "QQP\n",
      "---------\n",
      "Feliss Kappa: 0.19122512161909952\n",
      "RTE\n",
      "---------\n",
      "Feliss Kappa: 0.1799227762377962\n",
      "SST-2\n",
      "---------\n",
      "Feliss Kappa: 0.32452024723070516\n",
      "STS-B\n",
      "---------\n",
      "Feliss Kappa: 0.22154005502120921\n",
      "WNLI\n",
      "---------\n",
      "Feliss Kappa: -0.25000037252910756\n"
     ]
    }
   ],
   "source": [
    "fleiss_kappa_tasks_separate(heads)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MLPs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CoLA\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.3232398928813502\n",
      "Null hypothesis (all seeds are similar) is not rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 9\n",
      "Total mask pairs - 10\n",
      "Percentage - 0.9\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "CoLA-seed_1337,CoLA-seed_166\tCoLA-seed_1337,CoLA-seed_42\tCoLA-seed_1337,CoLA-seed_86\tCoLA-seed_166,CoLA-seed_42\tCoLA-seed_166,CoLA-seed_71\tCoLA-seed_166,CoLA-seed_86\tCoLA-seed_42,CoLA-seed_71\tCoLA-seed_42,CoLA-seed_86\tCoLA-seed_71,CoLA-seed_86\n",
      "\n",
      "\n",
      "\n",
      "HANS\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.9594907270068882\n",
      "Null hypothesis (all seeds are similar) is not rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 10\n",
      "Total mask pairs - 10\n",
      "Percentage - 1.0\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "HANS-seed_1337,HANS-seed_166\tHANS-seed_1337,HANS-seed_42\tHANS-seed_1337,HANS-seed_71\tHANS-seed_1337,HANS-seed_86\tHANS-seed_166,HANS-seed_42\tHANS-seed_166,HANS-seed_71\tHANS-seed_166,HANS-seed_86\tHANS-seed_42,HANS-seed_71\tHANS-seed_42,HANS-seed_86\tHANS-seed_71,HANS-seed_86\n",
      "\n",
      "\n",
      "\n",
      "HANS_MNLI\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.9786790480094321\n",
      "Null hypothesis (all seeds are similar) is not rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 10\n",
      "Total mask pairs - 10\n",
      "Percentage - 1.0\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "HANS_MNLI-seed_1337,HANS_MNLI-seed_166\tHANS_MNLI-seed_1337,HANS_MNLI-seed_42\tHANS_MNLI-seed_1337,HANS_MNLI-seed_71\tHANS_MNLI-seed_1337,HANS_MNLI-seed_86\tHANS_MNLI-seed_166,HANS_MNLI-seed_42\tHANS_MNLI-seed_166,HANS_MNLI-seed_71\tHANS_MNLI-seed_166,HANS_MNLI-seed_86\tHANS_MNLI-seed_42,HANS_MNLI-seed_71\tHANS_MNLI-seed_42,HANS_MNLI-seed_86\tHANS_MNLI-seed_71,HANS_MNLI-seed_86\n",
      "\n",
      "\n",
      "\n",
      "MNLI\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.8224157057672481\n",
      "Null hypothesis (all seeds are similar) is not rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 10\n",
      "Total mask pairs - 10\n",
      "Percentage - 1.0\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "MNLI-seed_1337,MNLI-seed_166\tMNLI-seed_1337,MNLI-seed_42\tMNLI-seed_1337,MNLI-seed_71\tMNLI-seed_1337,MNLI-seed_86\tMNLI-seed_166,MNLI-seed_42\tMNLI-seed_166,MNLI-seed_71\tMNLI-seed_166,MNLI-seed_86\tMNLI-seed_42,MNLI-seed_71\tMNLI-seed_42,MNLI-seed_86\tMNLI-seed_71,MNLI-seed_86\n",
      "\n",
      "\n",
      "\n",
      "MNLI_TWO\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.8312925312037189\n",
      "Null hypothesis (all seeds are similar) is not rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 10\n",
      "Total mask pairs - 10\n",
      "Percentage - 1.0\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "MNLI_TWO-seed_1337,MNLI_TWO-seed_166\tMNLI_TWO-seed_1337,MNLI_TWO-seed_42\tMNLI_TWO-seed_1337,MNLI_TWO-seed_71\tMNLI_TWO-seed_1337,MNLI_TWO-seed_86\tMNLI_TWO-seed_166,MNLI_TWO-seed_42\tMNLI_TWO-seed_166,MNLI_TWO-seed_71\tMNLI_TWO-seed_166,MNLI_TWO-seed_86\tMNLI_TWO-seed_42,MNLI_TWO-seed_71\tMNLI_TWO-seed_42,MNLI_TWO-seed_86\tMNLI_TWO-seed_71,MNLI_TWO-seed_86\n",
      "\n",
      "\n",
      "\n",
      "MNLI_TWO_HALF\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.5249309467861041\n",
      "Null hypothesis (all seeds are similar) is not rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 9\n",
      "Total mask pairs - 10\n",
      "Percentage - 0.9\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "MNLI_TWO_HALF-seed_1337,MNLI_TWO_HALF-seed_166\tMNLI_TWO_HALF-seed_1337,MNLI_TWO_HALF-seed_42\tMNLI_TWO_HALF-seed_1337,MNLI_TWO_HALF-seed_71\tMNLI_TWO_HALF-seed_1337,MNLI_TWO_HALF-seed_86\tMNLI_TWO_HALF-seed_166,MNLI_TWO_HALF-seed_42\tMNLI_TWO_HALF-seed_166,MNLI_TWO_HALF-seed_71\tMNLI_TWO_HALF-seed_42,MNLI_TWO_HALF-seed_71\tMNLI_TWO_HALF-seed_42,MNLI_TWO_HALF-seed_86\tMNLI_TWO_HALF-seed_71,MNLI_TWO_HALF-seed_86\n",
      "\n",
      "\n",
      "\n",
      "MRPC\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.6833711942656508\n",
      "Null hypothesis (all seeds are similar) is not rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 9\n",
      "Total mask pairs - 10\n",
      "Percentage - 0.9\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "MRPC-seed_1337,MRPC-seed_166\tMRPC-seed_1337,MRPC-seed_42\tMRPC-seed_1337,MRPC-seed_71\tMRPC-seed_1337,MRPC-seed_86\tMRPC-seed_166,MRPC-seed_71\tMRPC-seed_166,MRPC-seed_86\tMRPC-seed_42,MRPC-seed_71\tMRPC-seed_42,MRPC-seed_86\tMRPC-seed_71,MRPC-seed_86\n",
      "\n",
      "\n",
      "\n",
      "QNLI\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.9824769036935782\n",
      "Null hypothesis (all seeds are similar) is not rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 10\n",
      "Total mask pairs - 10\n",
      "Percentage - 1.0\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "QNLI-seed_1337,QNLI-seed_166\tQNLI-seed_1337,QNLI-seed_42\tQNLI-seed_1337,QNLI-seed_71\tQNLI-seed_1337,QNLI-seed_86\tQNLI-seed_166,QNLI-seed_42\tQNLI-seed_166,QNLI-seed_71\tQNLI-seed_166,QNLI-seed_86\tQNLI-seed_42,QNLI-seed_71\tQNLI-seed_42,QNLI-seed_86\tQNLI-seed_71,QNLI-seed_86\n",
      "\n",
      "\n",
      "\n",
      "QQP\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.9735009788392561\n",
      "Null hypothesis (all seeds are similar) is not rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 10\n",
      "Total mask pairs - 10\n",
      "Percentage - 1.0\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "QQP-seed_1337,QQP-seed_166\tQQP-seed_1337,QQP-seed_42\tQQP-seed_1337,QQP-seed_71\tQQP-seed_1337,QQP-seed_86\tQQP-seed_166,QQP-seed_42\tQQP-seed_166,QQP-seed_71\tQQP-seed_166,QQP-seed_86\tQQP-seed_42,QQP-seed_71\tQQP-seed_42,QQP-seed_86\tQQP-seed_71,QQP-seed_86\n",
      "\n",
      "\n",
      "\n",
      "RTE\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.25477265448360564\n",
      "Null hypothesis (all seeds are similar) is not rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 9\n",
      "Total mask pairs - 10\n",
      "Percentage - 0.9\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "RTE-seed_1337,RTE-seed_166\tRTE-seed_1337,RTE-seed_42\tRTE-seed_1337,RTE-seed_71\tRTE-seed_166,RTE-seed_42\tRTE-seed_166,RTE-seed_71\tRTE-seed_166,RTE-seed_86\tRTE-seed_42,RTE-seed_71\tRTE-seed_42,RTE-seed_86\tRTE-seed_71,RTE-seed_86\n",
      "\n",
      "\n",
      "\n",
      "SST-2\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.8979161202208832\n",
      "Null hypothesis (all seeds are similar) is not rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 10\n",
      "Total mask pairs - 10\n",
      "Percentage - 1.0\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "SST-2-seed_1337,SST-2-seed_166\tSST-2-seed_1337,SST-2-seed_42\tSST-2-seed_1337,SST-2-seed_71\tSST-2-seed_1337,SST-2-seed_86\tSST-2-seed_166,SST-2-seed_42\tSST-2-seed_166,SST-2-seed_71\tSST-2-seed_166,SST-2-seed_86\tSST-2-seed_42,SST-2-seed_71\tSST-2-seed_42,SST-2-seed_86\tSST-2-seed_71,SST-2-seed_86\n",
      "\n",
      "\n",
      "\n",
      "STS-B\n",
      "---------------------------------------------------------------------\n",
      "p-value: 0.47205433276263287\n",
      "Null hypothesis (all seeds are similar) is not rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 10\n",
      "Total mask pairs - 10\n",
      "Percentage - 1.0\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "STS-B-seed_1337,STS-B-seed_166\tSTS-B-seed_1337,STS-B-seed_42\tSTS-B-seed_1337,STS-B-seed_71\tSTS-B-seed_1337,STS-B-seed_86\tSTS-B-seed_166,STS-B-seed_42\tSTS-B-seed_166,STS-B-seed_71\tSTS-B-seed_166,STS-B-seed_86\tSTS-B-seed_42,STS-B-seed_71\tSTS-B-seed_42,STS-B-seed_86\tSTS-B-seed_71,STS-B-seed_86\n",
      "\n",
      "\n",
      "\n",
      "WNLI\n",
      "---------------------------------------------------------------------\n",
      "p-value: nan\n",
      "Null hypothesis (all seeds are similar) is not rejected.\n",
      "Total mask pairs where Null hypothesis is not rejected - 0\n",
      "Total mask pairs - 10\n",
      "Percentage - 0.0\n",
      "\n",
      "Similar Mask Pairs:\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print_p_value_tasks_separate(mlps, mask=\"mlp_mask\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CoLA\n",
      "---------\n",
      "alpha: 0.4537037037037036\n",
      "HANS\n",
      "---------\n",
      "alpha: 0.3413631022326674\n",
      "HANS_MNLI\n",
      "---------\n",
      "alpha: 0.31835686777920413\n",
      "MNLI\n",
      "---------\n",
      "alpha: 0.18055555555555558\n",
      "MNLI_TWO\n",
      "---------\n",
      "alpha: 0.2804878048780487\n",
      "MNLI_TWO_HALF\n",
      "---------\n",
      "alpha: 0.4319640564826701\n",
      "MRPC\n",
      "---------\n",
      "alpha: 0.03804347826086951\n",
      "QNLI\n",
      "---------\n",
      "alpha: 0.16193181818181823\n",
      "QQP\n",
      "---------\n",
      "alpha: 0.03146374829001364\n",
      "RTE\n",
      "---------\n",
      "alpha: -0.0993788819875776\n",
      "SST-2\n",
      "---------\n",
      "alpha: 0.06349206349206349\n",
      "STS-B\n",
      "---------\n",
      "alpha: 0.09870740305522918\n",
      "WNLI\n",
      "---------\n",
      "alpha: nan\n"
     ]
    }
   ],
   "source": [
    "krippendorff_alpha_tasks_separate(mlps, mask=\"mlp_mask\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pairwise Task to task comparison"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Heads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tasks = sorted(heads.keys())\n",
    "for t1, t2 in itertools.combinations(tasks, 2):\n",
    "    print_p_value_tasks(heads, [t1, t2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CoLA,HANS\n",
      "---------\n",
      "alpha: 0.12605966529226342\n",
      "CoLA,HANS_MNLI\n",
      "---------\n",
      "alpha: 0.15390838813984287\n",
      "CoLA,MNLI\n",
      "---------\n",
      "alpha: 0.1472484134521177\n",
      "CoLA,MNLI_TWO\n",
      "---------\n",
      "alpha: 0.14212075685326764\n",
      "CoLA,MNLI_TWO_HALF\n",
      "---------\n",
      "alpha: 0.14118472263574544\n",
      "CoLA,MRPC\n",
      "---------\n",
      "alpha: 0.13809026972718175\n",
      "CoLA,QNLI\n",
      "---------\n",
      "alpha: 0.17694109929350343\n",
      "CoLA,QQP\n",
      "---------\n",
      "alpha: 0.1261541914627099\n",
      "CoLA,RTE\n",
      "---------\n",
      "alpha: 0.135281470108642\n",
      "CoLA,SST-2\n",
      "---------\n",
      "alpha: 0.19279541253541732\n",
      "CoLA,STS-B\n",
      "---------\n",
      "alpha: 0.14422753716871406\n",
      "CoLA,WNLI\n",
      "---------\n",
      "alpha: -0.009957825396038844\n",
      "HANS,HANS_MNLI\n",
      "---------\n",
      "alpha: 0.17639652014652096\n",
      "HANS,MNLI\n",
      "---------\n",
      "alpha: 0.15112170220719234\n",
      "HANS,MNLI_TWO\n",
      "---------\n",
      "alpha: 0.1591999239058064\n",
      "HANS,MNLI_TWO_HALF\n",
      "---------\n",
      "alpha: 0.16878590515113767\n",
      "HANS,MRPC\n",
      "---------\n",
      "alpha: 0.14627980692708387\n",
      "HANS,QNLI\n",
      "---------\n",
      "alpha: 0.1579596739596747\n",
      "HANS,QQP\n",
      "---------\n",
      "alpha: 0.1233765941364734\n",
      "HANS,RTE\n",
      "---------\n",
      "alpha: 0.14104061594315065\n",
      "HANS,SST-2\n",
      "---------\n",
      "alpha: 0.17432937876915733\n",
      "HANS,STS-B\n",
      "---------\n",
      "alpha: 0.1442396064226641\n",
      "HANS,WNLI\n",
      "---------\n",
      "alpha: -0.009623983769964761\n",
      "HANS_MNLI,MNLI\n",
      "---------\n",
      "alpha: 0.21410759656909628\n",
      "HANS_MNLI,MNLI_TWO\n",
      "---------\n",
      "alpha: 0.24977425602146752\n",
      "HANS_MNLI,MNLI_TWO_HALF\n",
      "---------\n",
      "alpha: 0.24511561497862921\n",
      "HANS_MNLI,MRPC\n",
      "---------\n",
      "alpha: 0.18620522765319203\n",
      "HANS_MNLI,QNLI\n",
      "---------\n",
      "alpha: 0.22258034575995778\n",
      "HANS_MNLI,QQP\n",
      "---------\n",
      "alpha: 0.18723093673009883\n",
      "HANS_MNLI,RTE\n",
      "---------\n",
      "alpha: 0.1598100611772495\n",
      "HANS_MNLI,SST-2\n",
      "---------\n",
      "alpha: 0.17962345835051297\n",
      "HANS_MNLI,STS-B\n",
      "---------\n",
      "alpha: 0.16102426169357742\n",
      "HANS_MNLI,WNLI\n",
      "---------\n",
      "alpha: 0.013826452295703784\n",
      "MNLI,MNLI_TWO\n",
      "---------\n",
      "alpha: 0.24148033735412422\n",
      "MNLI,MNLI_TWO_HALF\n",
      "---------\n",
      "alpha: 0.260903228715729\n",
      "MNLI,MRPC\n",
      "---------\n",
      "alpha: 0.2018879741222478\n",
      "MNLI,QNLI\n",
      "---------\n",
      "alpha: 0.22794618133130917\n",
      "MNLI,QQP\n",
      "---------\n",
      "alpha: 0.15662409840858604\n",
      "MNLI,RTE\n",
      "---------\n",
      "alpha: 0.16903253609136015\n",
      "MNLI,SST-2\n",
      "---------\n",
      "alpha: 0.1866126679295409\n",
      "MNLI,STS-B\n",
      "---------\n",
      "alpha: 0.169926782911902\n",
      "MNLI,WNLI\n",
      "---------\n",
      "alpha: -0.007094333781319628\n",
      "MNLI_TWO,MNLI_TWO_HALF\n",
      "---------\n",
      "alpha: 0.26306518565491166\n",
      "MNLI_TWO,MRPC\n",
      "---------\n",
      "alpha: 0.19965727533457556\n",
      "MNLI_TWO,QNLI\n",
      "---------\n",
      "alpha: 0.24891389764734473\n",
      "MNLI_TWO,QQP\n",
      "---------\n",
      "alpha: 0.18882625482625526\n",
      "MNLI_TWO,RTE\n",
      "---------\n",
      "alpha: 0.18112303457375523\n",
      "MNLI_TWO,SST-2\n",
      "---------\n",
      "alpha: 0.18432870585242334\n",
      "MNLI_TWO,STS-B\n",
      "---------\n",
      "alpha: 0.19134753077724498\n",
      "MNLI_TWO,WNLI\n",
      "---------\n",
      "alpha: 0.014716413456108524\n",
      "MNLI_TWO_HALF,MRPC\n",
      "---------\n",
      "alpha: 0.22509217562872863\n",
      "MNLI_TWO_HALF,QNLI\n",
      "---------\n",
      "alpha: 0.27745294926505826\n",
      "MNLI_TWO_HALF,QQP\n",
      "---------\n",
      "alpha: 0.19279541253541754\n",
      "MNLI_TWO_HALF,RTE\n",
      "---------\n",
      "alpha: 0.21044439054625852\n",
      "MNLI_TWO_HALF,SST-2\n",
      "---------\n",
      "alpha: 0.17277563277563313\n",
      "MNLI_TWO_HALF,STS-B\n",
      "---------\n",
      "alpha: 0.18598579071206778\n",
      "MNLI_TWO_HALF,WNLI\n",
      "---------\n",
      "alpha: 0.010580063605767687\n",
      "MRPC,QNLI\n",
      "---------\n",
      "alpha: 0.2160456789450047\n",
      "MRPC,QQP\n",
      "---------\n",
      "alpha: 0.1550365571845289\n",
      "MRPC,RTE\n",
      "---------\n",
      "alpha: 0.17729586574222367\n",
      "MRPC,SST-2\n",
      "---------\n",
      "alpha: 0.14113742937816565\n",
      "MRPC,STS-B\n",
      "---------\n",
      "alpha: 0.20541939982580726\n",
      "MRPC,WNLI\n",
      "---------\n",
      "alpha: -0.009823219690904494\n",
      "QNLI,QQP\n",
      "---------\n",
      "alpha: 0.207452260125098\n",
      "QNLI,RTE\n",
      "---------\n",
      "alpha: 0.20714294384274767\n",
      "QNLI,SST-2\n",
      "---------\n",
      "alpha: 0.1388534623697263\n",
      "QNLI,STS-B\n",
      "---------\n",
      "alpha: 0.1853593746597867\n",
      "QNLI,WNLI\n",
      "---------\n",
      "alpha: 0.01338979632990589\n",
      "QQP,RTE\n",
      "---------\n",
      "alpha: 0.1339025659981542\n",
      "QQP,SST-2\n",
      "---------\n",
      "alpha: 0.13510883386106287\n",
      "QQP,STS-B\n",
      "---------\n",
      "alpha: 0.16080631686743785\n",
      "QQP,WNLI\n",
      "---------\n",
      "alpha: -0.006209416934876977\n",
      "RTE,SST-2\n",
      "---------\n",
      "alpha: 0.14867864245777163\n",
      "RTE,STS-B\n",
      "---------\n",
      "alpha: 0.19011358490598496\n",
      "RTE,WNLI\n",
      "---------\n",
      "alpha: -0.006435594101887654\n",
      "SST-2,STS-B\n",
      "---------\n",
      "alpha: 0.13794308159459434\n",
      "SST-2,WNLI\n",
      "---------\n",
      "alpha: 0.03264789079843444\n",
      "STS-B,WNLI\n",
      "---------\n",
      "alpha: 0.0018897860180763226\n"
     ]
    }
   ],
   "source": [
    "tasks = sorted(heads.keys())\n",
    "for t1, t2 in itertools.combinations(tasks, 2):\n",
    "    krippendorff_alpha_tasks(heads, [t1, t2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MLPs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for t1, t2 in itertools.combinations(tasks, 2):\n",
    "    print_p_value_tasks(mlps, [t1, t2], mask=\"mlp_mask\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CoLA,HANS\n",
      "---------\n",
      "alpha: 0.22340111995284406\n",
      "CoLA,HANS_MNLI\n",
      "---------\n",
      "alpha: 0.21501754385964922\n",
      "CoLA,MNLI\n",
      "---------\n",
      "alpha: 0.2309992283950617\n",
      "CoLA,MNLI_TWO\n",
      "---------\n",
      "alpha: 0.23728654970760243\n",
      "CoLA,MNLI_TWO_HALF\n",
      "---------\n",
      "alpha: 0.17047953216374256\n",
      "CoLA,MRPC\n",
      "---------\n",
      "alpha: 0.16700000000000004\n",
      "CoLA,QNLI\n",
      "---------\n",
      "alpha: 0.11688311688311681\n",
      "CoLA,QQP\n",
      "---------\n",
      "alpha: 0.1050849145873799\n",
      "CoLA,RTE\n",
      "---------\n",
      "alpha: 0.07444444444444442\n",
      "CoLA,SST-2\n",
      "---------\n",
      "alpha: 0.046904730297800845\n",
      "CoLA,STS-B\n",
      "---------\n",
      "alpha: 0.1933392278219862\n",
      "CoLA,WNLI\n",
      "---------\n",
      "alpha: -0.046202768424990825\n",
      "HANS,HANS_MNLI\n",
      "---------\n",
      "alpha: 0.29772079772079785\n",
      "HANS,MNLI\n",
      "---------\n",
      "alpha: 0.13049295049912513\n",
      "HANS,MNLI_TWO\n",
      "---------\n",
      "alpha: 0.2573599240265908\n",
      "HANS,MNLI_TWO_HALF\n",
      "---------\n",
      "alpha: 0.26543209876543206\n",
      "HANS,MRPC\n",
      "---------\n",
      "alpha: 0.12598140308983674\n",
      "HANS,QNLI\n",
      "---------\n",
      "alpha: 0.23404030811438215\n",
      "HANS,QQP\n",
      "---------\n",
      "alpha: 0.19013888888888886\n",
      "HANS,RTE\n",
      "---------\n",
      "alpha: 0.07431527913455627\n",
      "HANS,SST-2\n",
      "---------\n",
      "alpha: 0.13802498670919716\n",
      "HANS,STS-B\n",
      "---------\n",
      "alpha: 0.20759890325107722\n",
      "HANS,WNLI\n",
      "---------\n",
      "alpha: 0.039871196497702543\n",
      "HANS_MNLI,MNLI\n",
      "---------\n",
      "alpha: 0.11737038243062337\n",
      "HANS_MNLI,MNLI_TWO\n",
      "---------\n",
      "alpha: 0.30409356725146197\n",
      "HANS_MNLI,MNLI_TWO_HALF\n",
      "---------\n",
      "alpha: 0.30409356725146197\n",
      "HANS_MNLI,MRPC\n",
      "---------\n",
      "alpha: 0.11114981229923748\n",
      "HANS_MNLI,QNLI\n",
      "---------\n",
      "alpha: 0.2844444444444444\n",
      "HANS_MNLI,QQP\n",
      "---------\n",
      "alpha: 0.16923868312757184\n",
      "HANS_MNLI,RTE\n",
      "---------\n",
      "alpha: 0.05588451565463037\n",
      "HANS_MNLI,SST-2\n",
      "---------\n",
      "alpha: 0.1488194444444444\n",
      "HANS_MNLI,STS-B\n",
      "---------\n",
      "alpha: 0.18471035137701797\n",
      "HANS_MNLI,WNLI\n",
      "---------\n",
      "alpha: 0.01619155432060626\n",
      "MNLI,MNLI_TWO\n",
      "---------\n",
      "alpha: 0.17764752704511733\n",
      "MNLI,MNLI_TWO_HALF\n",
      "---------\n",
      "alpha: 0.13459242374905034\n",
      "MNLI,MRPC\n",
      "---------\n",
      "alpha: 0.051530934343434476\n",
      "MNLI,QNLI\n",
      "---------\n",
      "alpha: 0.05943152454780365\n",
      "MNLI,QQP\n",
      "---------\n",
      "alpha: 0.05333333333333323\n",
      "MNLI,RTE\n",
      "---------\n",
      "alpha: 0.051530934343434254\n",
      "MNLI,SST-2\n",
      "---------\n",
      "alpha: 0.058246280468502776\n",
      "MNLI,STS-B\n",
      "---------\n",
      "alpha: 0.0896710232925112\n",
      "MNLI,WNLI\n",
      "---------\n",
      "alpha: -0.017094017094017255\n",
      "MNLI_TWO,MNLI_TWO_HALF\n",
      "---------\n",
      "alpha: 0.32106689487947504\n",
      "MNLI_TWO,MRPC\n",
      "---------\n",
      "alpha: 0.13878246062154098\n",
      "MNLI_TWO,QNLI\n",
      "---------\n",
      "alpha: 0.2222222222222221\n",
      "MNLI_TWO,QQP\n",
      "---------\n",
      "alpha: 0.17798353909465014\n",
      "MNLI_TWO,RTE\n",
      "---------\n",
      "alpha: 0.04667363288052939\n",
      "MNLI_TWO,SST-2\n",
      "---------\n",
      "alpha: 0.14881944444444428\n",
      "MNLI_TWO,STS-B\n",
      "---------\n",
      "alpha: 0.16856600189933513\n",
      "MNLI_TWO,WNLI\n",
      "---------\n",
      "alpha: 0.008027168879283564\n",
      "MNLI_TWO_HALF,MRPC\n",
      "---------\n",
      "alpha: 0.14799334339564219\n",
      "MNLI_TWO_HALF,QNLI\n",
      "---------\n",
      "alpha: 0.29333333333333345\n",
      "MNLI_TWO_HALF,QQP\n",
      "---------\n",
      "alpha: 0.23045267489711918\n",
      "MNLI_TWO_HALF,RTE\n",
      "---------\n",
      "alpha: 0.028251867332326985\n",
      "MNLI_TWO_HALF,SST-2\n",
      "---------\n",
      "alpha: 0.24798611111111102\n",
      "MNLI_TWO_HALF,STS-B\n",
      "---------\n",
      "alpha: 0.19278252611585944\n",
      "MNLI_TWO_HALF,WNLI\n",
      "---------\n",
      "alpha: 0.040684710644574684\n",
      "MRPC,QNLI\n",
      "---------\n",
      "alpha: 0.06954732510288075\n",
      "MRPC,QQP\n",
      "---------\n",
      "alpha: 0.09423704240666908\n",
      "MRPC,RTE\n",
      "---------\n",
      "alpha: -0.036835748792270584\n",
      "MRPC,SST-2\n",
      "---------\n",
      "alpha: 0.06222222222222229\n",
      "MRPC,STS-B\n",
      "---------\n",
      "alpha: 0.11737038243062325\n",
      "MRPC,WNLI\n",
      "---------\n",
      "alpha: -0.05653479566523045\n",
      "QNLI,QQP\n",
      "---------\n",
      "alpha: 0.14799334339564219\n",
      "QNLI,RTE\n",
      "---------\n",
      "alpha: -0.00880658436213988\n",
      "QNLI,SST-2\n",
      "---------\n",
      "alpha: 0.1432034444082636\n",
      "QNLI,STS-B\n",
      "---------\n",
      "alpha: 0.12521543385740908\n",
      "QNLI,WNLI\n",
      "---------\n",
      "alpha: -0.028043593833067648\n",
      "QQP,RTE\n",
      "---------\n",
      "alpha: -0.04953485562401827\n",
      "QQP,SST-2\n",
      "---------\n",
      "alpha: 0.11738696334331755\n",
      "QQP,STS-B\n",
      "---------\n",
      "alpha: 0.10749999999999993\n",
      "QQP,WNLI\n",
      "---------\n",
      "alpha: -0.05027014329339918\n",
      "RTE,SST-2\n",
      "---------\n",
      "alpha: -0.07111111111111112\n",
      "RTE,STS-B\n",
      "---------\n",
      "alpha: 0.014038134520062195\n",
      "RTE,WNLI\n",
      "---------\n",
      "alpha: -0.07984071027549278\n",
      "SST-2,STS-B\n",
      "---------\n",
      "alpha: 0.05894471026049952\n",
      "SST-2,WNLI\n",
      "---------\n",
      "alpha: -0.03383630544124383\n",
      "STS-B,WNLI\n",
      "---------\n",
      "alpha: -0.020405948116791528\n"
     ]
    }
   ],
   "source": [
    "tasks = sorted(mlps.keys())\n",
    "for t1, t2 in itertools.combinations(tasks, 2):\n",
    "    krippendorff_alpha_tasks(mlps, [t1, t2], mask=\"mlp_mask\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "sets = []\n",
    "for seed in heads['MNLI']:\n",
    "    a = heads['MNLI'][seed]['head_mask'].reshape(-1)\n",
    "    sets.append({x[0] for x in np.argwhere(a == 1)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "union =  set.union(*sets)\n",
    "intersection = set.intersection(*sets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.291970802919708"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(intersection) / len(union)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.7 64-bit",
   "language": "python",
   "name": "python37764bit04702c7235a6448a9727054baaa32e98"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}