src/experiments/Evaluate_All_Experiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8a1282ec-de7b-4158-af05-a9df4dff3217",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import cleanlab\n",
    "from cleanlab.rank import get_label_quality_scores, get_label_quality_ensemble_scores\n",
    "from cleanlab.internal.label_quality_utils import get_normalized_entropy\n",
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from sklearn.metrics import roc_auc_score, f1_score, precision_score, recall_score, precision_recall_curve, accuracy_score, log_loss\n",
    "from matplotlib import pyplot as plt\n",
    "import copy\n",
    "\n",
    "sys.path.insert(0, \"../\")\n",
    "from eval_metrics import lift_at_k\n",
    "from active_learning_scores import least_confidence\n",
    "\n",
    "# experimental version of label quality ensemble scores with additional weighting schemes\n",
    "from label_quality_ensemble_scores_experimental import get_label_quality_ensemble_scores_experimental\n",
    "\n",
    "pd.set_option('display.max_rows', None)\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cc01f68-a877-4fc5-9ecc-a829b64a3254",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "?get_label_quality_ensemble_scores_experimental"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22cedb20-dd14-4d6b-8aec-084cf90fe419",
   "metadata": {},
   "source": [
    "## Models\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6250211d-cdb2-4379-98b8-f905260e2a8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "models = [\n",
    "    \"resnet18\",\n",
    "    \"resnet50d\",\n",
    "    \"efficientnet_b1\",\n",
    "    \"twins_pcpvt_base\",\n",
    "    \"swin_base_patch4_window7_224\",\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d10c1631-3e6d-46b1-88b1-51b8829a6cbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c6ba165-9cae-4507-93f1-19fc6accdc7a",
   "metadata": {},
   "source": [
    "## Dictionaries to map to display names\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0be6566b-76fe-4fcb-81b4-a47c2aefa0e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# dictionaries to map to display name\n",
    "\n",
    "method_adjust_pred_probs_display_dict = {\n",
    "    \"self_confidence-False\": \"Self Confidence\",\n",
    "    \"self_confidence-True\": \"Adjusted Self Confidence\",\n",
    "    \"normalized_margin-False\": \"Normalized Margin\",\n",
    "    \"normalized_margin-True\": \"Adjusted Normalized Margin\",\n",
    "    \"confidence_weighted_entropy-False\": \"Confidence Weighted Entropy\",\n",
    "    \"entropy-False\": \"Entropy\",\n",
    "    \"least_confidence-False\": \"Least Confidence\",\n",
    "}\n",
    "\n",
    "model_display_name_dict = {\n",
    "    \"swin_base_patch4_window7_224\": \"Swin Transformer\",\n",
    "    \"twins_pcpvt_base\": \"Twins PCPVT\",\n",
    "    \"efficientnet_b1\": \"EfficientNet-B1\",\n",
    "    \"resnet50d\": \"ResNet-50d\",\n",
    "    \"resnet18\": \"ResNet-18\",\n",
    "}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd1ba88c-3027-4b1d-98ae-a490565982de",
   "metadata": {},
   "source": [
    "## Load files from experiments\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52c25a9b-af7d-4c85-9614-2e71de4517ce",
   "metadata": {},
   "source": [
    "**Note:** we can refactor the code later to make it more concise but for now it reads the .npy files for each dataset within the for-loop\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a3d51ff1-9435-452f-a449-a52171084aa2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 17.7 s, sys: 2.47 s, total: 20.2 s\n",
      "Wall time: 20.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "experiments = []\n",
    "\n",
    "for model in models:\n",
    "\n",
    "    #### Andrew Ng DCAI Roman Numerals ####\n",
    "\n",
    "    # read numpy files\n",
    "    numpy_out_folder = f\"./roman-numeral/roman-numeral_train_val_dataset_cv_{model}/\"\n",
    "    pred_probs = np.load(numpy_out_folder + \"pred_probs.npy\")\n",
    "    labels = np.load(numpy_out_folder + \"labels.npy\")\n",
    "    images = np.load(numpy_out_folder + \"images.npy\", allow_pickle=True)\n",
    "    label_errors_mask = np.load(numpy_out_folder + \"label_errors_mask.npy\")\n",
    "\n",
    "    # store results of experiment\n",
    "    experiment_results = {\n",
    "        \"dataset\": \"roman-numeral\",\n",
    "        \"model\": model,\n",
    "        \"pred_probs\": pred_probs,\n",
    "        \"labels\": labels,\n",
    "        \"images\": images,\n",
    "        \"label_errors_mask\": label_errors_mask\n",
    "    }\n",
    "    experiments.append(experiment_results)\n",
    "\n",
    "\n",
    "    #### Cifar-10n-worst\n",
    "\n",
    "    # read numpy files\n",
    "    numpy_out_folder = f\"./cifar-10n-worst/cifar-10n-png_noise_type_worst_cv_{model}/\"\n",
    "    pred_probs = np.load(numpy_out_folder + \"pred_probs.npy\")\n",
    "    labels = np.load(numpy_out_folder + \"noisy_labels.npy\")\n",
    "    true_labels = np.load(numpy_out_folder + \"true_labels.npy\")\n",
    "    images = np.load(numpy_out_folder + \"images.npy\", allow_pickle=True)\n",
    "\n",
    "    label_errors_mask = (true_labels != labels) # boolean mask of label errors\n",
    "\n",
    "    # store results of experiment\n",
    "    experiment_results = {\n",
    "        \"dataset\": \"cifar-10n-worst\",\n",
    "        \"model\": model,\n",
    "        \"pred_probs\": pred_probs,\n",
    "        \"labels\": labels,\n",
    "        \"images\": images,\n",
    "        \"label_errors_mask\": label_errors_mask\n",
    "    }\n",
    "    experiments.append(experiment_results)\n",
    "\n",
    "\n",
    "    #### Cifar-10n-aggregate\n",
    "\n",
    "    # read numpy files\n",
    "    numpy_out_folder = f\"./cifar-10n-aggregate/cifar-10n-png_noise_type_aggre_cv_{model}/\"\n",
    "    pred_probs = np.load(numpy_out_folder + \"pred_probs.npy\")\n",
    "    labels = np.load(numpy_out_folder + \"noisy_labels.npy\")\n",
    "    true_labels = np.load(numpy_out_folder + \"true_labels.npy\")\n",
    "    images = np.load(numpy_out_folder + \"images.npy\", allow_pickle=True)\n",
    "\n",
    "    label_errors_mask = (true_labels != labels) # boolean mask of label errors\n",
    "\n",
    "    # store results of experiment\n",
    "    experiment_results = {\n",
    "        \"dataset\": \"cifar-10n-aggregate\",\n",
    "        \"model\": model,\n",
    "        \"pred_probs\": pred_probs,\n",
    "        \"labels\": labels,\n",
    "        \"images\": images,\n",
    "        \"label_errors_mask\": label_errors_mask    \n",
    "    }\n",
    "    experiments.append(experiment_results)\n",
    "\n",
    "\n",
    "    #### Cifar-10\n",
    "\n",
    "    # synthetic noise amount 20% and sparsity 40% (as defined in confident learning paper)\n",
    "\n",
    "    # read numpy files\n",
    "    numpy_out_folder = f\"./cifar-10/cifar10_train_dataset_noise_amount_0.2_sparsity_0.4_cv_{model}/\"\n",
    "    pred_probs = np.load(numpy_out_folder + \"pred_probs.npy\")\n",
    "    labels = np.load(numpy_out_folder + \"noisy_labels.npy\")\n",
    "    true_labels = np.load(numpy_out_folder + \"true_labels.npy\")\n",
    "    images = np.load(numpy_out_folder + \"images.npy\", allow_pickle=True)\n",
    "\n",
    "    label_errors_mask = (true_labels != labels) # boolean mask of label errors\n",
    "\n",
    "    # store results of experiment\n",
    "    experiment_results = {\n",
    "        \"dataset\": \"cifar-10\",\n",
    "        \"model\": model,\n",
    "        \"pred_probs\": pred_probs,\n",
    "        \"labels\": labels,\n",
    "        \"images\": images,\n",
    "        \"label_errors_mask\": label_errors_mask    \n",
    "    }\n",
    "    experiments.append(experiment_results)\n",
    "\n",
    "\n",
    "    #### Food-101n\n",
    "\n",
    "    # we only have verified labels for ~50K images so we have to evaluate within this subset\n",
    "\n",
    "    # read numpy files\n",
    "    numpy_out_folder = f\"./food-101n/food-101n_cv_{model}/\"\n",
    "    pred_probs = np.load(numpy_out_folder + \"pred_probs.npy\")\n",
    "    labels = np.load(numpy_out_folder + \"noisy_labels.npy\")\n",
    "    images = np.load(numpy_out_folder + \"images.npy\", allow_pickle=True)\n",
    "\n",
    "    # read verified training labels\n",
    "    path_verified_train = \"./food-101n/verified_train.tsv\"\n",
    "    df_verified_train = pd.read_csv(path_verified_train, sep='\\t')\n",
    "\n",
    "    # instantiate DataFrame with all training data\n",
    "    df_image_paths = pd.DataFrame({\n",
    "        \"class_name/key\": pd.Series(images).map(lambda f: \"/\".join(Path(f).parts[-2:]))\n",
    "    })\n",
    "\n",
    "    # join to append verification_label column\n",
    "    df_image_paths_w_verified = df_image_paths.merge(df_verified_train, on=\"class_name/key\", how=\"left\")\n",
    "\n",
    "    # subset of data with verified labels\n",
    "    verified_subset_mask = ~df_image_paths_w_verified.verification_label.isnull().values\n",
    "\n",
    "    # filter on verified subset\n",
    "    pred_probs = pred_probs[verified_subset_mask]\n",
    "    labels = labels[verified_subset_mask]\n",
    "    images = images[verified_subset_mask]\n",
    "\n",
    "    # boolean mask of label errors\n",
    "    label_errors_mask = df_image_paths_w_verified[\"verification_label\"].values[verified_subset_mask] == 0\n",
    "\n",
    "    # store results of experiment\n",
    "    experiment_results = {\n",
    "        \"dataset\": \"food-101n\",\n",
    "        \"model\": model,\n",
    "        \"pred_probs\": pred_probs,\n",
    "        \"labels\": labels,\n",
    "        \"images\": images,\n",
    "        \"label_errors_mask\": label_errors_mask  \n",
    "    }\n",
    "    experiments.append(experiment_results)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9809d96e-d97c-491c-9b95-cd16eafa9474",
   "metadata": {},
   "source": [
    "## Prepare data for ensemble model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a02558d5-7e95-4301-b9fb-ae6c01de8820",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "dataset_model_output = {}\n",
    "\n",
    "for i, experiment in enumerate(experiments):\n",
    "\n",
    "    # experiment results\n",
    "    dataset = experiment[\"dataset\"]\n",
    "    model = experiment[\"model\"]\n",
    "    pred_probs = experiment[\"pred_probs\"]\n",
    "    labels = experiment[\"labels\"]\n",
    "    images = experiment[\"images\"]\n",
    "    label_errors_target = experiment[\"label_errors_mask\"]\n",
    "\n",
    "    # check\n",
    "    if dataset not in dataset_model_output.keys():\n",
    "\n",
    "        # init list of pred_probs and labels\n",
    "        dataset_model_output[dataset] = {}\n",
    "        dataset_model_output[dataset][\"pred_probs_list\"] = []\n",
    "        dataset_model_output[dataset][\"labels_list\"] = []\n",
    "        dataset_model_output[dataset][\"images_list\"] = []\n",
    "        dataset_model_output[dataset][\"label_errors_target_list\"] = []\n",
    "\n",
    "    # store model output on dataset as key\n",
    "    dataset_model_output[dataset][\"pred_probs_list\"].append(pred_probs)\n",
    "    dataset_model_output[dataset][\"labels_list\"].append(labels)\n",
    "    dataset_model_output[dataset][\"images_list\"].append(images)\n",
    "    dataset_model_output[dataset][\"label_errors_target_list\"].append(\n",
    "        label_errors_target\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19dc1736-d9d7-4b31-ac4f-6f5f6592df8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb640ead-3c3a-4b67-a280-0198941b7e41",
   "metadata": {},
   "source": [
    "## Evaluate ensemble model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c2f12ea-20a8-4c32-9cae-d17e35d96d8b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d533022-66d7-4c74-bf70-1321ea6d5edc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "238e97a8-736f-492a-8b03-71ec0ed81060",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# args to pass to get_label_quality_scores()\n",
    "score_params = \\\n",
    "[\n",
    "    (\"self_confidence\", False),\n",
    "    (\"self_confidence\", True),\n",
    "    (\"normalized_margin\", False),\n",
    "    (\"normalized_margin\", True),\n",
    "    (\"confidence_weighted_entropy\", False)\n",
    "]\n",
    "\n",
    "\n",
    "ensemble_evaluations = []\n",
    "\n",
    "dataset_best_weights = []\n",
    "\n",
    "for dataset_key in dataset_model_output.keys():\n",
    "\n",
    "    # get list of pred_probs, labels for dataset\n",
    "    pred_probs_list = dataset_model_output[dataset_key][\"pred_probs_list\"]\n",
    "    labels_list = dataset_model_output[dataset_key][\"labels_list\"]\n",
    "    images_list = dataset_model_output[dataset_key][\"images_list\"]\n",
    "    label_errors_target_list = dataset_model_output[dataset_key][\"label_errors_target_list\"]\n",
    "    \n",
    "    # use for sanity check (noisy labels and images from each model should be the same because they were generated from the same cross-val procedure\n",
    "    for i, (labels_temp, images_temp) in enumerate(zip(labels_list, images_list)):\n",
    "\n",
    "        if i == 0:\n",
    "            labels_temp_previous = copy.deepcopy(labels_temp)\n",
    "            images_temp_previous = copy.deepcopy(images_temp)       \n",
    "        else:\n",
    "            assert (labels_temp_previous == labels_temp).all()\n",
    "            assert (images_temp_previous == images_temp).all()    \n",
    "    \n",
    "    # take the first (the others are the same)\n",
    "    labels = labels_list[0]\n",
    "    label_errors_target = label_errors_target_list[0]\n",
    "    \n",
    "    # compute accuracy\n",
    "    accuracy_list = []\n",
    "    for pred_probs in pred_probs_list:\n",
    "        \n",
    "        # accuracy of single model\n",
    "        accuracy = (pred_probs.argmax(axis=1) == labels).mean()\n",
    "        accuracy_list.append(accuracy)\n",
    "        \n",
    "    # accuracy weights\n",
    "    acc_weights = np.array(accuracy_list) / sum(accuracy_list)    \n",
    "    \n",
    "    # average predictions\n",
    "    pred_probs_avg = sum(pred_probs_list) / len(pred_probs_list)\n",
    "    \n",
    "    #### can refactor below to a function that accepts weights and pred_probs_list\n",
    "    \n",
    "    # accuracy-weighted predictions\n",
    "    pred_probs_avg_acc_weighted = sum([acc_weights[i] * p for i, p in enumerate(pred_probs_list)])\n",
    "    \n",
    "\n",
    "    #### find best t in T for exp-log-loss weighting\n",
    "    T = [1e-4, 1e-3, 1e-2, 1e-1, 1e0, 1e1, 1e2, 2e2]\n",
    "\n",
    "    pred_probs_avg_log_loss_weighted = None\n",
    "    inv_log_loss_weights = None\n",
    "    best_eval_log_loss = float(\"inf\")\n",
    "    best_t = None\n",
    "\n",
    "    for t in T:\n",
    "\n",
    "        log_loss_list = []\n",
    "\n",
    "        # pred_probs for each model\n",
    "        for pred_probs in pred_probs_list:\n",
    "            log_loss_ = np.exp(t * (-log_loss(labels, pred_probs)))\n",
    "            log_loss_list.append(log_loss_)\n",
    "\n",
    "        # weights using log loss\n",
    "        inv_log_loss_weights_temp = np.array(log_loss_list) / sum(log_loss_list)\n",
    "\n",
    "        # weighted average\n",
    "        pred_probs_avg_log_loss_weighted_temp = sum([inv_log_loss_weights_temp[i] * p for i, p in enumerate(pred_probs_list)])\n",
    "\n",
    "        # evaluate log_loss with this weighted average\n",
    "        eval_log_loss = log_loss(labels, pred_probs_avg_log_loss_weighted_temp)\n",
    "\n",
    "\n",
    "        # check if this is the best eval_log_loss so far\n",
    "        if best_eval_log_loss > eval_log_loss:\n",
    "            best_eval_log_loss = eval_log_loss\n",
    "            best_t = t\n",
    "            pred_probs_avg_log_loss_weighted = pred_probs_avg_log_loss_weighted_temp.copy()\n",
    "            inv_log_loss_weights = inv_log_loss_weights_temp.copy()    \n",
    "\n",
    "    df_temp = pd.DataFrame({\n",
    "        \"dataset\": dataset_key,\n",
    "        \"models\": models,\n",
    "        \"best_t\": best_t,\n",
    "        \"best_eval_log_loss\": best_eval_log_loss,\n",
    "        \"inv_log_loss_weights\": inv_log_loss_weights,\n",
    "    })\n",
    "    \n",
    "    # save the weights for analysis later\n",
    "    dataset_best_weights.append(df_temp)\n",
    "            \n",
    "    print()\n",
    "    print(dataset_key)\n",
    "    print(best_eval_log_loss)\n",
    "    print(inv_log_loss_weights)\n",
    "    print(pred_probs_avg_log_loss_weighted)\n",
    "    \n",
    "    \n",
    "    #### label quality scoring\n",
    "    \n",
    "    for score_param in score_params:\n",
    "        \n",
    "        # label quality scoring method\n",
    "        method, adjust_pred_probs = score_param\n",
    "    \n",
    "        # compute scores\n",
    "        \n",
    "        # use average pred_probs\n",
    "        label_quality_scores_avg = get_label_quality_scores(labels=labels, pred_probs=pred_probs_avg, method=method, adjust_pred_probs=adjust_pred_probs)\n",
    "        \n",
    "        # use average pred_probs weighted by accuracy\n",
    "        label_quality_scores_avg_acc_weighted = get_label_quality_scores(labels=labels, pred_probs=pred_probs_avg_acc_weighted, method=method, adjust_pred_probs=adjust_pred_probs)\n",
    "        \n",
    "        # use average pred_probs weighted by log loss\n",
    "        label_quality_scores_avg_log_loss_weighted = get_label_quality_scores(labels=labels, pred_probs=pred_probs_avg_log_loss_weighted, method=method, adjust_pred_probs=adjust_pred_probs)\n",
    "        \n",
    "        # use pred_probs_list (weighted by accuracy)\n",
    "        label_quality_scores_agg_acc = get_label_quality_ensemble_scores_experimental(\n",
    "            labels=labels, \n",
    "            pred_probs_list=pred_probs_list, \n",
    "            method=method, \n",
    "            adjust_pred_probs=adjust_pred_probs, \n",
    "            verbose=0,\n",
    "            weight_ensemble_members_by=\"accuracy\"\n",
    "        )\n",
    "\n",
    "        # use pred_probs_list (uniform_weights)\n",
    "        label_quality_scores_agg_uni = get_label_quality_ensemble_scores_experimental(\n",
    "            labels=labels, \n",
    "            pred_probs_list=pred_probs_list, \n",
    "            method=method, \n",
    "            adjust_pred_probs=adjust_pred_probs,\n",
    "            verbose=0,\n",
    "            weight_ensemble_members_by=\"uniform\"\n",
    "        )\n",
    "        \n",
    "        # use pred_probs_list (weight by inverse log loss)\n",
    "        label_quality_scores_agg_log_loss = get_label_quality_ensemble_scores_experimental(\n",
    "            labels=labels, \n",
    "            pred_probs_list=pred_probs_list, \n",
    "            method=method, \n",
    "            adjust_pred_probs=adjust_pred_probs,\n",
    "            verbose=0, \n",
    "            weight_ensemble_members_by=\"custom\",\n",
    "            custom_weights=inv_log_loss_weights # custom weights!\n",
    "        )        \n",
    "        \n",
    "        # compute accuracy of detecting label errors\n",
    "        auroc_avg = roc_auc_score(label_errors_target, 1 - label_quality_scores_avg)\n",
    "        auroc_avg_acc_weighted = roc_auc_score(label_errors_target, 1 - label_quality_scores_avg_acc_weighted)\n",
    "        auroc_avg_log_loss_weighted = roc_auc_score(label_errors_target, 1 - label_quality_scores_avg_log_loss_weighted)        \n",
    "        \n",
    "        auroc_agg_acc = roc_auc_score(label_errors_target, 1 - label_quality_scores_agg_acc)\n",
    "        auroc_agg_uni = roc_auc_score(label_errors_target, 1 - label_quality_scores_agg_uni)\n",
    "        auroc_agg_log_loss = roc_auc_score(label_errors_target, 1 - label_quality_scores_agg_log_loss)        \n",
    "        \n",
    "        # lift at K where K = number of label errors\n",
    "        lift_at_num_label_errors_avg = lift_at_k(label_errors_target, 1 - label_quality_scores_avg, k=label_errors_target.sum())\n",
    "        lift_at_num_label_errors_avg_acc_weighted = lift_at_k(label_errors_target, 1 - label_quality_scores_avg_acc_weighted, k=label_errors_target.sum())\n",
    "        lift_at_num_label_errors_avg_log_loss_weighted = lift_at_k(label_errors_target, 1 - label_quality_scores_avg_log_loss_weighted, k=label_errors_target.sum())\n",
    "        \n",
    "        lift_at_num_label_errors_agg_acc = lift_at_k(label_errors_target, 1 - label_quality_scores_agg_acc, k=label_errors_target.sum())\n",
    "        lift_at_num_label_errors_agg_uni = lift_at_k(label_errors_target, 1 - label_quality_scores_agg_uni, k=label_errors_target.sum())\n",
    "        lift_at_num_label_errors_agg_log_loss = lift_at_k(label_errors_target, 1 - label_quality_scores_agg_log_loss, k=label_errors_target.sum())        \n",
    "        \n",
    "        # lift at k=100\n",
    "        lift_at_100_avg = lift_at_k(label_errors_target, 1 - label_quality_scores_avg, k=100)\n",
    "        lift_at_100_avg_acc_weighted = lift_at_k(label_errors_target, 1 - label_quality_scores_avg_acc_weighted, k=100)\n",
    "        lift_at_100_avg_log_loss_weighted = lift_at_k(label_errors_target, 1 - label_quality_scores_avg_log_loss_weighted, k=100)\n",
    "        \n",
    "        lift_at_100_agg_acc = lift_at_k(label_errors_target, 1 - label_quality_scores_agg_acc, k=100)\n",
    "        lift_at_100_agg_uni = lift_at_k(label_errors_target, 1 - label_quality_scores_agg_uni, k=100)\n",
    "        lift_at_100_agg_log_loss = lift_at_k(label_errors_target, 1 - label_quality_scores_agg_log_loss, k=100)        \n",
    "\n",
    "        ensemble_evaluation_results_avg = {\n",
    "            \"ensemble_method\": \"avg_pred_probs\",\n",
    "            \"method\": method,\n",
    "            \"adjust_pred_probs\": adjust_pred_probs,\n",
    "            \"dataset\": dataset_key,\n",
    "            \"model\": \"ensemble\",\n",
    "            \"dataset_num_samples\": labels.shape[0],\n",
    "            \"dataset_num_label_errors\": label_errors_target.sum(),\n",
    "            \"auroc\": auroc_avg,\n",
    "            \"lift_at_num_label_errors\": lift_at_num_label_errors_avg,\n",
    "            \"lift_at_100\": lift_at_100_avg\n",
    "        }\n",
    "        \n",
    "        ensemble_evaluation_results_avg_acc_weighted = {\n",
    "            \"ensemble_method\": \"avg_pred_probs_weighted_by_accuracy\",\n",
    "            \"method\": method,\n",
    "            \"adjust_pred_probs\": adjust_pred_probs,\n",
    "            \"dataset\": dataset_key,\n",
    "            \"model\": \"ensemble\",\n",
    "            \"dataset_num_samples\": labels.shape[0],\n",
    "            \"dataset_num_label_errors\": label_errors_target.sum(),\n",
    "            \"auroc\": auroc_avg_acc_weighted,\n",
    "            \"lift_at_num_label_errors\": lift_at_num_label_errors_avg_acc_weighted,\n",
    "            \"lift_at_100\": lift_at_100_avg_acc_weighted\n",
    "        }        \n",
    "        \n",
    "        ensemble_evaluation_results_avg_log_loss_weighted = {\n",
    "            \"ensemble_method\": \"avg_pred_probs_weighted_by_inv_log_loss\",\n",
    "            \"method\": method,\n",
    "            \"adjust_pred_probs\": adjust_pred_probs,\n",
    "            \"dataset\": dataset_key,\n",
    "            \"model\": \"ensemble\",\n",
    "            \"dataset_num_samples\": labels.shape[0],\n",
    "            \"dataset_num_label_errors\": label_errors_target.sum(),\n",
    "            \"auroc\": auroc_avg_log_loss_weighted,\n",
    "            \"lift_at_num_label_errors\": lift_at_num_label_errors_avg_log_loss_weighted,\n",
    "            \"lift_at_100\": lift_at_100_avg_log_loss_weighted\n",
    "        }                \n",
    "        \n",
    "        ensemble_evaluation_results_agg_acc = {\n",
    "            \"ensemble_method\": \"avg_scores_weighted_by_accuracy\",\n",
    "            \"method\": method,\n",
    "            \"adjust_pred_probs\": adjust_pred_probs,\n",
    "            \"dataset\": dataset_key,\n",
    "            \"model\": \"ensemble\",\n",
    "            \"dataset_num_samples\": labels.shape[0],\n",
    "            \"dataset_num_label_errors\": label_errors_target.sum(),\n",
    "            \"auroc\": auroc_agg_acc,\n",
    "            \"lift_at_num_label_errors\": lift_at_num_label_errors_agg_acc,\n",
    "            \"lift_at_100\": lift_at_100_agg_acc\n",
    "        }\n",
    "        \n",
    "        ensemble_evaluation_results_agg_uni = {\n",
    "            \"ensemble_method\": \"avg_scores\",\n",
    "            \"method\": method,\n",
    "            \"adjust_pred_probs\": adjust_pred_probs,\n",
    "            \"dataset\": dataset_key,\n",
    "            \"model\": \"ensemble\",\n",
    "            \"dataset_num_samples\": labels.shape[0],\n",
    "            \"dataset_num_label_errors\": label_errors_target.sum(),\n",
    "            \"auroc\": auroc_agg_uni,\n",
    "            \"lift_at_num_label_errors\": lift_at_num_label_errors_agg_uni,\n",
    "            \"lift_at_100\": lift_at_100_agg_uni\n",
    "        }\n",
    "        \n",
    "        ensemble_evaluation_results_agg_log_loss = {\n",
    "            \"ensemble_method\": \"avg_scores_weighted_by_inv_log_loss\",\n",
    "            \"method\": method,\n",
    "            \"adjust_pred_probs\": adjust_pred_probs,\n",
    "            \"dataset\": dataset_key,\n",
    "            \"model\": \"ensemble\",\n",
    "            \"dataset_num_samples\": labels.shape[0],\n",
    "            \"dataset_num_label_errors\": label_errors_target.sum(),\n",
    "            \"auroc\": auroc_agg_log_loss,\n",
    "            \"lift_at_num_label_errors\": lift_at_num_label_errors_agg_log_loss,\n",
    "            \"lift_at_100\": lift_at_100_agg_log_loss\n",
    "        }\n",
    "        \n",
    "        # store evaluation results\n",
    "        ensemble_evaluations.append(ensemble_evaluation_results_avg)\n",
    "        ensemble_evaluations.append(ensemble_evaluation_results_avg_acc_weighted)\n",
    "        ensemble_evaluations.append(ensemble_evaluation_results_avg_log_loss_weighted)\n",
    "        \n",
    "        ensemble_evaluations.append(ensemble_evaluation_results_agg_acc)\n",
    "        ensemble_evaluations.append(ensemble_evaluation_results_agg_uni)\n",
    "        ensemble_evaluations.append(ensemble_evaluation_results_agg_log_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9a50a83-6ec8-4eaf-b4a5-a04b5df9f25d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataset_best_weights_w_model = []\n",
    "\n",
    "# for df_temp in dataset_best_weights:\n",
    "#     df_temp[\"models\"] = models\n",
    "    \n",
    "#     dataset_best_weights_w_model.append(df_temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8868fbd7-79a9-4714-8728-5cbb6f8bb802",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_dataset_best_weights = pd.concat(dataset_best_weights_w_model)\n",
    "\n",
    "df_dataset_best_weights.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a09bf774-76c6-40a9-b542-35f921b24749",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5086753-0ef9-4584-a030-32ed1933eb15",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "507714e8-82f0-41dc-a116-4c1b7a63ba17",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_evaluations_ensemble = pd.DataFrame(ensemble_evaluations)\n",
    "\n",
    "\n",
    "df_evaluations_ensemble[\"method_adjust_pred_probs\"] = (\n",
    "    df_evaluations_ensemble.method\n",
    "    + \"-\"\n",
    "    + df_evaluations_ensemble.adjust_pred_probs.astype(str)\n",
    ")\n",
    "df_evaluations_ensemble[\"dataset_model\"] = (\n",
    "    df_evaluations_ensemble.dataset + \" | \" + df_evaluations_ensemble.model\n",
    ")\n",
    "\n",
    "df_evaluations_ensemble[\n",
    "    \"scoring_method\"\n",
    "] = df_evaluations_ensemble.method_adjust_pred_probs.map(\n",
    "    lambda x: method_adjust_pred_probs_display_dict[x]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4691ad6d-898e-416e-86f1-74f01e04914b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_evaluations_ensemble.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02ec35dd-325e-4b50-94d4-e1b69e88d06b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_evaluations_ensemble.groupby(\"ensemble_method\")[\"ensemble_method\"].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76df8a88-19ee-4404-851e-2433512c1071",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_evaluations_ensemble.to_csv(\"evaluation_ensemble.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fea02e40-22b9-44f3-961b-61641e63d239",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79c770e4-8cb4-46e7-9f6f-60131581f3d4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_evaluations_ensemble_pivot = pd.pivot_table(\n",
    "    df_evaluations_ensemble,\n",
    "    values=\"auroc\",\n",
    "    index=[\"scoring_method\", \"ensemble_method\"],\n",
    "    columns=[\"dataset\"],\n",
    ").reset_index()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9709aa87-44e6-4065-8dfa-c0b78ccbbdb3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_evaluations_ensemble_pivot.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "907c23ab-01c1-45ff-ae1f-121bf6241c83",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_evaluations_ensemble_auroc = (\n",
    "    pd.pivot_table(\n",
    "        df_evaluations_ensemble,\n",
    "        values=\"auroc\",\n",
    "        index=[\"dataset\", \"scoring_method\"],\n",
    "        columns=[\"ensemble_method\"],\n",
    "    )\n",
    "    .reset_index()\n",
    "    .sort_values(by=[\"dataset\", \"scoring_method\"])\n",
    ")\n",
    "\n",
    "df_evaluations_ensemble_lift_at_num_errors = (\n",
    "    pd.pivot_table(\n",
    "        df_evaluations_ensemble,\n",
    "        values=\"lift_at_num_label_errors\",\n",
    "        index=[\"dataset\", \"scoring_method\"],\n",
    "        columns=[\"ensemble_method\"],\n",
    "    )\n",
    "    .reset_index()\n",
    "    .sort_values(by=[\"dataset\", \"scoring_method\"])\n",
    ")\n",
    "\n",
    "df_evaluations_ensemble_lift_at_100 = (\n",
    "    pd.pivot_table(\n",
    "        df_evaluations_ensemble,\n",
    "        values=\"lift_at_100\",\n",
    "        index=[\"dataset\", \"scoring_method\"],\n",
    "        columns=[\"ensemble_method\"],\n",
    "    )\n",
    "    .reset_index()\n",
    "    .sort_values(by=[\"dataset\", \"scoring_method\"])\n",
    ")\n",
    "\n",
    "df_evaluations_ensemble_auroc[\"dataset_scoring_method\"] = \\\n",
    "    df_evaluations_ensemble_auroc.dataset + \" | \" + df_evaluations_ensemble_auroc.scoring_method\n",
    "\n",
    "df_evaluations_ensemble_lift_at_num_errors[\"dataset_scoring_method\"] = \\\n",
    "    df_evaluations_ensemble_lift_at_num_errors.dataset + \" | \" + df_evaluations_ensemble_lift_at_num_errors.scoring_method\n",
    "\n",
    "df_evaluations_ensemble_lift_at_100[\"dataset_scoring_method\"] = \\\n",
    "    df_evaluations_ensemble_lift_at_100.dataset + \" | \" + df_evaluations_ensemble_lift_at_100.scoring_method\n",
    "\n",
    "df_evaluations_ensemble_lift_at_100.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a087b696-e2fe-4e25-a9ad-43e653e84e65",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_evaluations_ensemble_auroc.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27af5783-d15b-4834-aede-00e9c478f27d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "# NOTE: this requires evaluating scores for single model (e.g. Swin Transformer)\n",
    "# Code to evaluate single model is further down in this notebook\n",
    "# Final version of notebook have cells in the right order so folks can run cells in the right sequence\n",
    "\n",
    "df_evaluations_swin = df_evaluations[df_evaluations.model_name == \"Swin Transformer\"]\n",
    "\n",
    "df_evaluations_swin_auroc = (\n",
    "    pd.pivot_table(\n",
    "        df_evaluations_swin,\n",
    "        values=\"auroc\",\n",
    "        index=[\"dataset\", \"scoring_method\"],\n",
    "        columns=[\"model_name\"],\n",
    "    )\n",
    "    .reset_index()\n",
    "    .sort_values(by=[\"dataset\", \"scoring_method\"])\n",
    ")\n",
    "\n",
    "df_evaluations_swin_lift_at_num_errors = (\n",
    "    pd.pivot_table(\n",
    "        df_evaluations_swin,\n",
    "        values=\"lift_at_num_label_errors\",\n",
    "        index=[\"dataset\", \"scoring_method\"],\n",
    "        columns=[\"model_name\"],\n",
    "    )\n",
    "    .reset_index()\n",
    "    .sort_values(by=[\"dataset\", \"scoring_method\"])\n",
    ")\n",
    "\n",
    "df_evaluations_swin_lift_at_100 = (\n",
    "    pd.pivot_table(\n",
    "        df_evaluations_swin,\n",
    "        values=\"lift_at_100\",\n",
    "        index=[\"dataset\", \"scoring_method\"],\n",
    "        columns=[\"model_name\"],\n",
    "    )\n",
    "    .reset_index()\n",
    "    .sort_values(by=[\"dataset\", \"scoring_method\"])\n",
    ")\n",
    "\n",
    "df_evaluations_swin_lift_at_100.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f8ee03c-a4c5-4b6e-8b6b-3a9c70c44a52",
   "metadata": {},
   "outputs": [],
   "source": [
    "# combine ensemble results with single best model results (Swin Transformer)\n",
    "\n",
    "df_evaluations_ensemble_auroc = \\\n",
    "    df_evaluations_ensemble_auroc.merge(df_evaluations_swin_auroc, on=[\"dataset\", \"scoring_method\"], how=\"left\")\n",
    "\n",
    "df_evaluations_ensemble_lift_at_num_errors = \\\n",
    "    df_evaluations_ensemble_lift_at_num_errors.merge(df_evaluations_swin_lift_at_num_errors, on=[\"dataset\", \"scoring_method\"], how=\"left\")\n",
    "\n",
    "df_evaluations_ensemble_lift_at_100 = \\\n",
    "    df_evaluations_ensemble_lift_at_100.merge(df_evaluations_swin_lift_at_100, on=[\"dataset\", \"scoring_method\"], how=\"left\")\n",
    "\n",
    "df_evaluations_ensemble_lift_at_num_errors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e10cdb55-cec1-41cb-b760-daf9a45726a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_evaluations_ensemble_auroc.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12a97ef4-0613-4528-a2ae-e0eac1a8a8da",
   "metadata": {},
   "outputs": [],
   "source": [
    "title = \"AUROC\"\n",
    "\n",
    "# pointer\n",
    "df = df_evaluations_ensemble_auroc.sort_values(by=[\"dataset\", \"avg_scores_weighted_by_inv_log_loss\"])\n",
    "# df = df[df.scoring_method == \"Self Confidence\"]\n",
    "\n",
    "labels = df[\"dataset_scoring_method\"].tolist()\n",
    "x = np.arange(len(labels))  # the label locations\n",
    "\n",
    "\n",
    "s0 = df[\"Swin Transformer\"].tolist()\n",
    "\n",
    "s1 = df[\"avg_pred_probs\"].tolist()\n",
    "s2 = df[\"avg_pred_probs_weighted_by_accuracy\"].tolist()\n",
    "s3 = df[\"avg_pred_probs_weighted_by_inv_log_loss\"].tolist()\n",
    "\n",
    "s4 = df[\"avg_scores\"].tolist()\n",
    "s5 = df[\"avg_scores_weighted_by_accuracy\"].tolist()\n",
    "s6 = df[\"avg_scores_weighted_by_inv_log_loss\"].tolist()\n",
    "\n",
    "jf = 0.15 # jitter factor\n",
    "markersize = 8\n",
    "alpha = 1\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "_ = ax.plot(s0, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Swin Transformer\", markersize=markersize, alpha=alpha)\n",
    "_ = ax.plot(s1, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Avg Preds\", markersize=markersize, alpha=alpha)\n",
    "# _ = ax.plot(s2, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Avg Preds (Weighted By Accuracy)\", markersize=markersize, alpha=alpha)\n",
    "# _ = ax.plot(s3, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Avg Preds (Weighted By Exp(-T*log_loss)\", markersize=markersize, alpha=alpha)\n",
    "# _ = ax.plot(s4, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Avg Scores\", markersize=markersize, alpha=alpha)\n",
    "# _ = ax.plot(s5, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Avg Scores (Weighted By Accuracy)\", markersize=markersize, alpha=alpha)\n",
    "_ = ax.plot(s6, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Weighted Avg Scores (Weighted By Exponential Negative Log Loss)\", markersize=markersize, alpha=alpha)\n",
    "\n",
    "# _ = ax.plot(s6, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Avg Scores (Weighted By Exp(-T*log_loss)\", markersize=markersize, alpha=alpha)\n",
    "\n",
    "\n",
    "\n",
    "# Add some text for labels, title and custom x-axis tick labels, etc.\n",
    "# ax.set_xlabel(\"\")\n",
    "ax.set_title(title, fontsize=24, fontweight=\"bold\")\n",
    "# ax.set_title(\"Dataset | Scoring Method\", fontsize=12, fontweight=\"bold\")\n",
    "ax.set_yticks(x, labels)\n",
    "ax.tick_params(axis='both', which='major', labelsize=12)\n",
    "ax.legend(fontsize=12)\n",
    "# fig.tight_layout()\n",
    "plt.rcParams[\"figure.figsize\"] = (15, 10)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11e2a273-6d0c-43f9-a9d1-569db2f626e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "title = \"Lift at # Errors\"\n",
    "\n",
    "# pointer\n",
    "df = df_evaluations_ensemble_lift_at_num_errors.sort_values(by=[\"dataset\", \"avg_scores_weighted_by_inv_log_loss\"])\n",
    "\n",
    "labels = df[\"dataset_scoring_method\"].tolist()\n",
    "x = np.arange(len(labels))  # the label locations\n",
    "\n",
    "\n",
    "s0 = df[\"Swin Transformer\"].tolist()\n",
    "\n",
    "s1 = df[\"avg_pred_probs\"].tolist()\n",
    "s2 = df[\"avg_pred_probs_weighted_by_accuracy\"].tolist()\n",
    "s3 = df[\"avg_pred_probs_weighted_by_inv_log_loss\"].tolist()\n",
    "\n",
    "s4 = df[\"avg_scores\"].tolist()\n",
    "s5 = df[\"avg_scores_weighted_by_accuracy\"].tolist()\n",
    "s6 = df[\"avg_scores_weighted_by_inv_log_loss\"].tolist()\n",
    "\n",
    "jf = 0.15 # jitter factor\n",
    "markersize = 8\n",
    "alpha = 1\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "_ = ax.plot(s0, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Swin Transformer\", markersize=markersize, alpha=alpha)\n",
    "_ = ax.plot(s1, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Avg Preds\", markersize=markersize, alpha=alpha)\n",
    "# _ = ax.plot(s2, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Avg Preds (Weighted By Accuracy)\", markersize=markersize, alpha=alpha)\n",
    "# _ = ax.plot(s3, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Avg Preds (Weighted By Exp(-T*log_loss)\", markersize=markersize, alpha=alpha)\n",
    "# _ = ax.plot(s4, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Avg Scores\", markersize=markersize, alpha=alpha)\n",
    "# _ = ax.plot(s5, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Avg Scores (Weighted By Accuracy)\", markersize=markersize, alpha=alpha)\n",
    "_ = ax.plot(s6, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Weighted Avg Scores (Weighted By Exponential Negative Log Loss)\", markersize=markersize, alpha=alpha)\n",
    "\n",
    "\n",
    "# _ = ax.plot(s6, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Avg Scores (Weighted By Exp(-T*log_loss)\", markersize=markersize, alpha=alpha)\n",
    "\n",
    "\n",
    "# Add some text for labels, title and custom x-axis tick labels, etc.\n",
    "# ax.set_xlabel(\"\")\n",
    "ax.set_title(title, fontsize=24, fontweight=\"bold\")\n",
    "# ax.set_title(\"Dataset | Scoring Method\", fontsize=12, fontweight=\"bold\")\n",
    "ax.set_yticks(x, labels)\n",
    "ax.tick_params(axis='both', which='major', labelsize=12)\n",
    "ax.legend(fontsize=12)\n",
    "# fig.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b8f09d3-3f08-454c-acb9-04dbc6a5ec66",
   "metadata": {},
   "outputs": [],
   "source": [
    "title = \"Lift at 100\"\n",
    "\n",
    "# pointer\n",
    "df = df_evaluations_ensemble_lift_at_100.sort_values(by=[\"dataset\", \"avg_scores_weighted_by_inv_log_loss\"])\n",
    "\n",
    "labels = df[\"dataset_scoring_method\"].tolist()\n",
    "x = np.arange(len(labels))  # the label locations\n",
    "\n",
    "\n",
    "s0 = df[\"Swin Transformer\"].tolist()\n",
    "\n",
    "s1 = df[\"avg_pred_probs\"].tolist()\n",
    "s2 = df[\"avg_pred_probs_weighted_by_accuracy\"].tolist()\n",
    "s3 = df[\"avg_pred_probs_weighted_by_inv_log_loss\"].tolist()\n",
    "\n",
    "s4 = df[\"avg_scores\"].tolist()\n",
    "s5 = df[\"avg_scores_weighted_by_accuracy\"].tolist()\n",
    "s6 = df[\"avg_scores_weighted_by_inv_log_loss\"].tolist()\n",
    "\n",
    "jf = 0.15 # jitter factor\n",
    "markersize = 8\n",
    "alpha = 1\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "_ = ax.plot(s0, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Swin Transformer\", markersize=markersize, alpha=alpha)\n",
    "_ = ax.plot(s1, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Avg Preds\", markersize=markersize, alpha=alpha)\n",
    "# _ = ax.plot(s2, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Avg Preds (Weighted By Accuracy)\", markersize=markersize, alpha=alpha)\n",
    "# _ = ax.plot(s3, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Avg Preds (Weighted By Exp(-T*log_loss)\", markersize=markersize, alpha=alpha)\n",
    "# _ = ax.plot(s4, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Avg Scores\", markersize=markersize, alpha=alpha)\n",
    "# _ = ax.plot(s5, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Avg Scores (Weighted By Accuracy)\", markersize=markersize, alpha=alpha)\n",
    "_ = ax.plot(s6, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Weighted Avg Scores (Weighted By Exponential Negative Log Loss)\", markersize=markersize, alpha=alpha)\n",
    "\n",
    "# _ = ax.plot(s6, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Ensemble: Avg Scores (Weighted By Exp(-T*log_loss)\", markersize=markersize, alpha=alpha)\n",
    "\n",
    "\n",
    "# Add some text for labels, title and custom x-axis tick labels, etc.\n",
    "# ax.set_xlabel(\"\")\n",
    "ax.set_title(title, fontsize=24, fontweight=\"bold\")\n",
    "# ax.set_title(\"Dataset | Scoring Method\", fontsize=12, fontweight=\"bold\")\n",
    "ax.set_yticks(x, labels)\n",
    "ax.tick_params(axis='both', which='major', labelsize=12)\n",
    "ax.legend(fontsize=12)\n",
    "# fig.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a85eac60-5905-41f2-9531-4e768a22b3aa",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "cfd35eb6-5141-4469-85bb-cec93400820b",
   "metadata": {},
   "source": [
    "## Evaluate all experiments (all dataset-models)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "630fd843-76fd-4b96-baee-2ec52b86d25b",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# args to pass to get_label_quality_scores()\n",
    "score_params = \\\n",
    "[\n",
    "    (\"self_confidence\", False),\n",
    "    (\"self_confidence\", True),\n",
    "    (\"normalized_margin\", False),\n",
    "    (\"normalized_margin\", True),\n",
    "    (\"confidence_weighted_entropy\", False)\n",
    "]\n",
    "\n",
    "evaluations = []\n",
    "precision_recall_curves = [] # store this separately\n",
    "accuracy_list = []\n",
    "\n",
    "for experiment in experiments:\n",
    "    \n",
    "    # experiment results\n",
    "    dataset = experiment[\"dataset\"]\n",
    "    model = experiment[\"model\"]\n",
    "    pred_probs = experiment[\"pred_probs\"]\n",
    "    labels = experiment[\"labels\"]\n",
    "    images = experiment[\"images\"]\n",
    "    label_errors_target = experiment[\"label_errors_mask\"]\n",
    "    \n",
    "    accuracy = {\n",
    "        \"dataset\": dataset,\n",
    "        \"model\": model,\n",
    "        \"cv_accuracy\": (pred_probs.argmax(axis=1) == labels).mean()\n",
    "    }\n",
    "    \n",
    "    accuracy_list.append(accuracy)\n",
    "    \n",
    "    #### label quality scoring\n",
    "    \n",
    "    for score_param in score_params:\n",
    "        \n",
    "        # label quality scoring method\n",
    "        method, adjust_pred_probs = score_param    \n",
    "    \n",
    "        # compute scores\n",
    "        label_quality_scores = get_label_quality_scores(labels=labels, pred_probs=pred_probs, method=method, adjust_pred_probs=adjust_pred_probs)\n",
    "\n",
    "        # compute accuracy of detecting label errors\n",
    "        auroc = roc_auc_score(label_errors_target, 1 - label_quality_scores)\n",
    "        \n",
    "        # lift at K where K = number of label errors\n",
    "        lift_at_num_label_errors = lift_at_k(label_errors_target, 1 - label_quality_scores, k=label_errors_target.sum())\n",
    "        \n",
    "        # lift at k=100\n",
    "        lift_at_100 = lift_at_k(label_errors_target, 1 - label_quality_scores, k=100)\n",
    "\n",
    "        evaluation_results = {\n",
    "            \"method\": method,\n",
    "            \"adjust_pred_probs\": adjust_pred_probs,\n",
    "            \"dataset\": dataset,\n",
    "            \"model\": model,\n",
    "            \"dataset_num_samples\": labels.shape[0],\n",
    "            \"dataset_num_label_errors\": label_errors_target.sum(),\n",
    "            \"auroc\": auroc,\n",
    "            \"lift_at_num_label_errors\": lift_at_num_label_errors,\n",
    "            \"lift_at_100\": lift_at_100            \n",
    "        }\n",
    "\n",
    "        # store evaluation results\n",
    "        evaluations.append(evaluation_results)\n",
    "        \n",
    "\n",
    "        # compute precision-recall curve using label quality scores\n",
    "        precision, recall, thresholds = precision_recall_curve(label_errors_target, 1 - label_quality_scores)        \n",
    "        \n",
    "        precision_recall_curve_results = {\n",
    "            \"method\": method,\n",
    "            \"adjust_pred_probs\": adjust_pred_probs,\n",
    "            \"dataset\": dataset,\n",
    "            \"model\": model,\n",
    "            \"label_quality_scores\": label_quality_scores,\n",
    "            \"precision\": precision,\n",
    "            \"recall\": recall,\n",
    "            \"thresholds\": thresholds\n",
    "        }\n",
    "        \n",
    "        # store precision-recall curve results\n",
    "        precision_recall_curves.append(precision_recall_curve_results)\n",
    "        \n",
    "\n",
    "    #### active learning scores to use as comparison\n",
    "    \n",
    "    al_scoring_funcs = {\n",
    "        \"entropy\": get_normalized_entropy,\n",
    "        \"least_confidence\": least_confidence\n",
    "    }\n",
    "    \n",
    "    for al_method in al_scoring_funcs.keys():\n",
    "        \n",
    "        # active learning scoring function\n",
    "        scoring_func = al_scoring_funcs[al_method]\n",
    "    \n",
    "        # score\n",
    "        al_scores = scoring_func(pred_probs)\n",
    "\n",
    "        # compute accuracy of detecting label errors\n",
    "        auroc = roc_auc_score(label_errors_target, al_scores)\n",
    "\n",
    "        # lift at K where K = number of label errors\n",
    "        lift_at_num_label_errors = lift_at_k(label_errors_target, al_scores, k=label_errors_target.sum())\n",
    "        \n",
    "        # lift at k=100\n",
    "        lift_at_100 = lift_at_k(label_errors_target, al_scores, k=100)\n",
    "\n",
    "        evaluation_results = {\n",
    "            \"method\": al_method,\n",
    "            \"adjust_pred_probs\": False,\n",
    "            \"dataset\": dataset,\n",
    "            \"model\": model,\n",
    "            \"dataset_num_samples\": labels.shape[0],\n",
    "            \"dataset_num_label_errors\": label_errors_target.sum(),\n",
    "            \"auroc\": auroc,\n",
    "            \"lift_at_num_label_errors\": lift_at_num_label_errors,\n",
    "            \"lift_at_100\": lift_at_100,\n",
    "        }\n",
    "\n",
    "        # store evaluation results\n",
    "        evaluations.append(evaluation_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ef35920-ad1e-4a50-a288-b4e28e764773",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# cross-validation accuracy\n",
    "df_cv_accuracy = pd.DataFrame(accuracy_list)\n",
    "\n",
    "df_cv_accuracy_pivot = (\n",
    "    pd.pivot_table(\n",
    "        df_cv_accuracy, values=\"cv_accuracy\", index=[\"model\"], columns=[\"dataset\"]\n",
    "    )\n",
    "    .reset_index()\n",
    "    .sort_values(by=\"roman-numeral\", ascending=False)\n",
    ")\n",
    "\n",
    "df_cv_accuracy_pivot[\"model\"] = df_cv_accuracy_pivot.model.map(\n",
    "    lambda x: model_display_name_dict[x]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "405f84b8-4aac-4b71-960c-8c799328398e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_cv_accuracy_pivot\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98bf223d-c80d-476d-8eb4-d53816c32f90",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_cv_accuracy.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a40e789-aacb-4acb-8915-98cafce31059",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3ece077-677a-4f44-822a-63b9ecd07adf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# master table with AUROC and Lift at K evaluation metrics for all methods, datasets, and models\n",
    "df_evaluations = pd.DataFrame(evaluations)\n",
    "\n",
    "# append cv accuracy\n",
    "df_evaluations = df_evaluations.merge(df_cv_accuracy, how=\"left\", on=[\"dataset\", \"model\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af0b457b-319e-4ac9-8c04-29cef4ed0db2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_evaluations[\"method_adjust_pred_probs\"] = (\n",
    "    df_evaluations.method + \"-\" + df_evaluations.adjust_pred_probs.astype(str)\n",
    ")\n",
    "df_evaluations[\"dataset_model\"] = df_evaluations.dataset + \" | \" + df_evaluations.model\n",
    "\n",
    "df_evaluations[\"scoring_method\"] = df_evaluations.method_adjust_pred_probs.map(\n",
    "    lambda x: method_adjust_pred_probs_display_dict[x]\n",
    ")\n",
    "df_evaluations[\"model_name\"] = df_evaluations.model.map(\n",
    "    lambda x: model_display_name_dict[x]\n",
    ")\n",
    "\n",
    "\n",
    "df_evaluations[\"model_name_w_acc\"] = df_evaluations.model_name + \" (\" + df_evaluations.cv_accuracy.round(4).astype(str) + \") \"\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35c7305c-1ec8-4282-9bed-621202b32598",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_evaluations.to_csv(\"evaluation_all_experiments.csv\")\n",
    "\n",
    "df_evaluations.head(3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56e65f28-39a9-41d3-ad37-b902ad1fe3f3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e75da212-9751-4744-b2a2-654248f09f67",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_evaluations_auroc = (\n",
    "    pd.pivot_table(\n",
    "        df_evaluations,\n",
    "        values=\"auroc\",\n",
    "        index=[\"dataset\", \"model_name\", \"model_name_w_acc\"],\n",
    "        columns=[\"scoring_method\"],\n",
    "    )\n",
    "    .reset_index()\n",
    "    .sort_values(by=[\"dataset\", \"model_name\"])\n",
    ")\n",
    "\n",
    "df_evaluations_lift_at_num_errors = (\n",
    "    pd.pivot_table(\n",
    "        df_evaluations,\n",
    "        values=\"lift_at_num_label_errors\",\n",
    "        index=[\"dataset\", \"model_name\", \"model_name_w_acc\"],\n",
    "        columns=[\"scoring_method\"],\n",
    "    )\n",
    "    .reset_index()\n",
    "    .sort_values(by=[\"dataset\", \"model_name\"])\n",
    ")\n",
    "\n",
    "df_evaluations_lift_at_100 = (\n",
    "    pd.pivot_table(\n",
    "        df_evaluations,\n",
    "        values=\"lift_at_100\",\n",
    "        index=[\"dataset\", \"model_name\", \"model_name_w_acc\"],\n",
    "        columns=[\"scoring_method\"],\n",
    "    )\n",
    "    .reset_index()\n",
    "    .sort_values(by=[\"dataset\", \"model_name\"])\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6f27401-031d-47a6-8b40-69fd0e216841",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_evaluations_auroc.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "408aa973-281b-42b6-863e-23cdb4cd5e05",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# pointer\n",
    "df = df_evaluations_auroc\n",
    "\n",
    "# Draw plot\n",
    "plt.figure(figsize=(10, 10), dpi=80)\n",
    "\n",
    "s = 60\n",
    "alpha = 0.9\n",
    "marker = \"o\"\n",
    "\n",
    "s0 = plt.scatter(\n",
    "    df[\"confidence_weighted_entropy-False\"], df.index, s=s, alpha=alpha, marker=marker\n",
    ")\n",
    "s1 = plt.scatter(df[\"self_confidence-False\"], df.index, s=s, alpha=alpha, marker=marker)\n",
    "s2 = plt.scatter(df[\"self_confidence-True\"], df.index, s=s, alpha=alpha, marker=marker)\n",
    "s3 = plt.scatter(\n",
    "    df[\"normalized_margin-False\"], df.index, s=s, alpha=alpha, marker=marker\n",
    ")\n",
    "s4 = plt.scatter(\n",
    "    df[\"normalized_margin-True\"], df.index, s=s, alpha=alpha, marker=marker\n",
    ")\n",
    "# s5 = plt.scatter(df[\"entropy-False\"], df.index, s=s, alpha=alpha, marker=marker)\n",
    "# s6 = plt.scatter(df[\"least_confidence-False\"], df.index, s=s, alpha=alpha, marker=marker)\n",
    "\n",
    "# for x, y, tex in zip(df[\"confidence_weighted_entropy-False\"], df.index, df[\"confidence_weighted_entropy-False\"]):\n",
    "#     t = plt.text(x, y, round(tex, 1), horizontalalignment='center',\n",
    "#                  verticalalignment='center', fontdict={'color':'white'})\n",
    "\n",
    "plt.title(\"AUROC\", fontsize=18)\n",
    "plt.yticks(df.index, df.dataset_model)\n",
    "plt.legend(\n",
    "    (s0, s1, s2, s3, s4),\n",
    "    (\n",
    "        \"Confidence Weighted Entropy (False)\",\n",
    "        \"Self Confidence (False)\",\n",
    "        \"Self Confidence (True)\",\n",
    "        \"Normalized Margin (False)\",\n",
    "        \"Normalized Margin (True)\",\n",
    "    ),\n",
    "    loc=\"upper center\",\n",
    "    bbox_to_anchor=(0.5, 1.2),\n",
    "    ncol=3,\n",
    "    fancybox=True,\n",
    "    shadow=True,\n",
    "    fontsize=12,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fca16522-9a82-400d-98f4-cc43bf0b96a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "title = \"AUROC\"\n",
    "\n",
    "# pointer\n",
    "df = df_evaluations_auroc.sort_values(by=[\"dataset\", \"Self Confidence\"])\n",
    "df[\"dataset_model\"] = df.dataset + \" | \" + df.model_name_w_acc\n",
    "\n",
    "labels = df[\"dataset_model\"].tolist()\n",
    "x = np.arange(len(labels))  # the label locations\n",
    "\n",
    "s0 = df[\"Confidence Weighted Entropy\"].tolist()\n",
    "s1 = df[\"Self Confidence\"].tolist()\n",
    "s2 = df[\"Adjusted Self Confidence\"].tolist()\n",
    "s3 = df[\"Normalized Margin\"].tolist()\n",
    "s4 = df[\"Adjusted Normalized Margin\"].tolist()\n",
    "\n",
    "\n",
    "jf = 0.15 # jitter factor\n",
    "markersize = 8\n",
    "alpha = 1\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "_ = ax.plot(s0, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Confidence Weighted Entropy\", markersize=markersize, alpha=alpha)\n",
    "_ = ax.plot(s1, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Self Confidence\", markersize=markersize, alpha=alpha)\n",
    "_ = ax.plot(s2, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Adjusted Self Confidence\", markersize=markersize, alpha=alpha)\n",
    "_ = ax.plot(s3, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Normalized Margin\", markersize=markersize, alpha=alpha)\n",
    "_ = ax.plot(s4, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Adjusted Normalized Margin\", markersize=markersize, alpha=alpha)\n",
    "\n",
    "# Add some text for labels, title and custom x-axis tick labels, etc.\n",
    "# ax.set_xlabel(\"\")\n",
    "ax.set_title(title, fontsize=24, fontweight=\"bold\")\n",
    "ax.set_yticks(x, labels)\n",
    "ax.tick_params(axis='both', which='major', labelsize=12)\n",
    "ax.legend(fontsize=12)\n",
    "# fig.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dbf8a1db-08a2-4a30-b308-f7654c420ae9",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "title = \"Lift at # Errors\"\n",
    "\n",
    "# pointer\n",
    "df = df_evaluations_lift_at_num_errors.sort_values(by=[\"dataset\", \"Self Confidence\"])\n",
    "df[\"dataset_model\"] = df.dataset + \" | \" + df.model_name_w_acc\n",
    "\n",
    "labels = df[\"dataset_model\"].tolist()\n",
    "x = np.arange(len(labels))  # the label locations\n",
    "\n",
    "s0 = df[\"Confidence Weighted Entropy\"].tolist()\n",
    "s1 = df[\"Self Confidence\"].tolist()\n",
    "s2 = df[\"Adjusted Self Confidence\"].tolist()\n",
    "s3 = df[\"Normalized Margin\"].tolist()\n",
    "s4 = df[\"Adjusted Normalized Margin\"].tolist()\n",
    "\n",
    "\n",
    "jf = 0.15 # jitter factor\n",
    "markersize = 8\n",
    "alpha = 1\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "_ = ax.plot(s0, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Confidence Weighted Entropy\", markersize=markersize, alpha=alpha)\n",
    "_ = ax.plot(s1, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Self Confidence\", markersize=markersize, alpha=alpha)\n",
    "_ = ax.plot(s2, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Adjusted Self Confidence\", markersize=markersize, alpha=alpha)\n",
    "_ = ax.plot(s3, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Normalized Margin\", markersize=markersize, alpha=alpha)\n",
    "_ = ax.plot(s4, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Adjusted Normalized Margin\", markersize=markersize, alpha=alpha)\n",
    "\n",
    "# Add some text for labels, title and custom x-axis tick labels, etc.\n",
    "# ax.set_xlabel(\"\")\n",
    "ax.set_title(title, fontsize=24, fontweight=\"bold\")\n",
    "ax.set_yticks(x, labels)\n",
    "ax.tick_params(axis='both', which='major', labelsize=12)\n",
    "ax.legend(fontsize=12)\n",
    "# fig.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "278f836c-9d49-408c-ad2f-9c460bfad1af",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "title = \"Lift at 100\"\n",
    "\n",
    "# pointer\n",
    "df = df_evaluations_lift_at_100.sort_values(by=[\"dataset\", \"Confidence Weighted Entropy\"])\n",
    "df[\"dataset_model\"] = df.dataset + \" | \" + df.model_name_w_acc\n",
    "\n",
    "labels = df[\"dataset_model\"].tolist()\n",
    "x = np.arange(len(labels))  # the label locations\n",
    "\n",
    "s0 = df[\"Confidence Weighted Entropy\"].tolist()\n",
    "s1 = df[\"Self Confidence\"].tolist()\n",
    "s2 = df[\"Adjusted Self Confidence\"].tolist()\n",
    "s3 = df[\"Normalized Margin\"].tolist()\n",
    "s4 = df[\"Adjusted Normalized Margin\"].tolist()\n",
    "\n",
    "\n",
    "jf = 0.15 # jitter factor\n",
    "markersize = 8\n",
    "alpha = 1\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "_ = ax.plot(s0, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Confidence Weighted Entropy\", markersize=markersize, alpha=alpha)\n",
    "_ = ax.plot(s1, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Self Confidence\", markersize=markersize, alpha=alpha)\n",
    "_ = ax.plot(s2, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Adjusted Self Confidence\", markersize=markersize, alpha=alpha)\n",
    "_ = ax.plot(s3, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Normalized Margin\", markersize=markersize, alpha=alpha)\n",
    "_ = ax.plot(s4, x + np.random.uniform(-jf, jf), marker=\"o\", linestyle=\"None\", label=\"Adjusted Normalized Margin\", markersize=markersize, alpha=alpha)\n",
    "\n",
    "# Add some text for labels, title and custom x-axis tick labels, etc.\n",
    "# ax.set_xlabel(\"\")\n",
    "ax.set_title(title, fontsize=24, fontweight=\"bold\")\n",
    "ax.set_yticks(x, labels)\n",
    "ax.tick_params(axis='both', which='major', labelsize=12)\n",
    "ax.legend(fontsize=12)\n",
    "# fig.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "860bfb17-778b-4780-8e34-392169897d8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_evaluations_lift_at_100.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4466d5d6-f291-452b-a88a-7e5f04e9ff55",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "87d8cb02-4441-4256-85b2-f334e02013b1",
   "metadata": {},
   "source": [
    "## Summarize evaluation results across all datasets-models\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a7cf3f5-8e48-4593-a579-8d2bfe046733",
   "metadata": {},
   "outputs": [],
   "source": [
    "def df_min_max_norm_group_by(\n",
    "    df: pd.DataFrame,\n",
    "    group_by_cols: list,\n",
    "    val_col: str,\n",
    ") -> pd.DataFrame:\n",
    "    \"\"\"\n",
    "    Create new column with min-max norm of a value column grouped by list of columns\n",
    "    \"\"\"\n",
    "\n",
    "    # calculate the max and min for each group\n",
    "    _max = df.groupby(group_by_cols)[val_col].transform(\"max\")\n",
    "    _min = df.groupby(group_by_cols)[val_col].transform(\"min\")\n",
    "\n",
    "    # calculate min-max norm for each group\n",
    "\n",
    "    df[f\"min_max_norm_by_{'_'.join(group_by_cols)}_{val_col}\"] = (\n",
    "        df[val_col] - _min\n",
    "    ) / (_max - _min)\n",
    "\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee69ee2e-5062-406e-b550-2bd6422e6338",
   "metadata": {},
   "outputs": [],
   "source": [
    "# take the average of min-max normalized metrics (min-max normalization is done separately for each dataset-model)\n",
    "df_eval_results = df_evaluations\n",
    "df_eval_results = df_min_max_norm_group_by(\n",
    "    df=df_eval_results, group_by_cols=[\"dataset\", \"model\"], val_col=\"auroc\"\n",
    ")\n",
    "df_eval_results = df_min_max_norm_group_by(\n",
    "    df=df_eval_results,\n",
    "    group_by_cols=[\"dataset\", \"model\"],\n",
    "    val_col=\"lift_at_num_label_errors\",\n",
    ")\n",
    "df_eval_results = df_min_max_norm_group_by(\n",
    "    df=df_eval_results, group_by_cols=[\"dataset\", \"model\"], val_col=\"lift_at_100\"\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb657028-9015-4a9c-b61f-a86854944cea",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_eval_results.head(7)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37faa7ac-821a-4980-8f52-9e7fc4526de4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d55e40c1-08ac-455b-993c-9e587264fc1d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60da8472-032c-4bb5-8d36-b7a666a0de0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#### aggregate evaluation metrics across all dataset-model pairs\n",
    "\n",
    "df_eval_results_agg = (\n",
    "    df_eval_results.groupby([\"scoring_method\"])\n",
    "    .agg(\n",
    "        avg_min_max_norm_by_dataset_model_auroc=(\n",
    "            \"min_max_norm_by_dataset_model_auroc\",\n",
    "            \"mean\",\n",
    "        ),\n",
    "        avg_min_max_norm_by_dataset_model_lift_at_num_label_errors=(\n",
    "            \"min_max_norm_by_dataset_model_lift_at_num_label_errors\",\n",
    "            \"mean\",\n",
    "        ),\n",
    "        avg_min_max_norm_by_dataset_model_lift_at_100=(\n",
    "            \"min_max_norm_by_dataset_model_lift_at_100\",\n",
    "            \"mean\",\n",
    "        ),\n",
    "    )\n",
    "    .reset_index()\n",
    "    .sort_values(by=\"avg_min_max_norm_by_dataset_model_auroc\", ascending=False)\n",
    ")\n",
    "\n",
    "\n",
    "df_eval_results_agg.to_csv(\"evaluation_all_experiments_aggregated.csv\")\n",
    "\n",
    "df_eval_results_agg\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51ac3f69-8577-4df6-a3f8-2f03fa2900be",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf7bf9c0-9271-4bc8-b8f3-612189172888",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d61fd09-7f70-4867-8136-6ec3175f2dc2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "20e90336-5e74-4200-9a3e-529fbaf71f5c",
   "metadata": {},
   "source": [
    "## Evaluation metrics for single best model (swin transformer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "161cfe6e-1520-4b66-8b16-1c3e326d0fb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_eval_results_swin = df_eval_results[\n",
    "    df_eval_results.model == \"swin_base_patch4_window7_224\"\n",
    "]\n",
    "df_eval_results_swin.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5465ee07-9b4b-4184-a114-1b42cad6b45c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_eval_results_swin_auroc = pd.pivot_table(\n",
    "    df_eval_results_swin, values=\"auroc\", index=[\"scoring_method\"], columns=[\"dataset\"]\n",
    ").reset_index()\n",
    "\n",
    "df_eval_results_swin_auroc\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c212c587-fbe4-4804-8cbb-743b6054a451",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_eval_results_swin_lift_at_num_errors = pd.pivot_table(\n",
    "    df_eval_results_swin,\n",
    "    values=\"lift_at_num_label_errors\",\n",
    "    index=[\"scoring_method\"],\n",
    "    columns=[\"dataset\"],\n",
    ").reset_index()\n",
    "\n",
    "df_eval_results_swin_lift_at_num_errors\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d15c880b-f8fc-41b8-8e8f-7cbb0813cd44",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_eval_results_swin_lift_at_100 = pd.pivot_table(\n",
    "    df_eval_results_swin,\n",
    "    values=\"lift_at_100\",\n",
    "    index=[\"scoring_method\"],\n",
    "    columns=[\"dataset\"],\n",
    ").reset_index()\n",
    "\n",
    "df_eval_results_swin_lift_at_100\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d35b005a-d89f-47f8-b704-2d387ffb2a48",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b5e5d9d-fde3-483f-90f3-8cbbf3088cc5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "cb5049eb-d8e1-407e-8ec3-b57abc241af2",
   "metadata": {},
   "source": [
    "## Evaluate filter_by options for model with overall best cross-val accuracy (swin transformer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "812a923c-fb20-48ee-8286-1cc390b11416",
   "metadata": {},
   "outputs": [],
   "source": [
    "# only use one selected model to evaluate filter_by options\n",
    "selected_model = \"swin_base_patch4_window7_224\"\n",
    "\n",
    "# Find label issues with different filter_by options\n",
    "filter_by_list = [\n",
    "    \"prune_by_noise_rate\",\n",
    "    \"prune_by_class\",\n",
    "    \"both\",\n",
    "    \"confident_learning\",\n",
    "    \"predicted_neq_given\",\n",
    "]\n",
    "\n",
    "results = []\n",
    "\n",
    "for experiment in experiments:\n",
    "\n",
    "    # experiment results\n",
    "    dataset = experiment[\"dataset\"]\n",
    "    model = experiment[\"model\"]\n",
    "    pred_probs = experiment[\"pred_probs\"]\n",
    "    labels = experiment[\"labels\"]\n",
    "    images = experiment[\"images\"]\n",
    "    label_errors_target = experiment[\"label_errors_mask\"]\n",
    "\n",
    "    # only run for selected model\n",
    "    if model == selected_model:\n",
    "        print(model)\n",
    "\n",
    "        for filter_by in filter_by_list:\n",
    "\n",
    "            # Find label issues\n",
    "            label_issues = cleanlab.filter.find_label_issues(\n",
    "                labels=labels, pred_probs=pred_probs, filter_by=filter_by\n",
    "            )\n",
    "\n",
    "            precision = precision_score(label_errors_target, label_issues)\n",
    "            recall = recall_score(label_errors_target, label_issues)\n",
    "            f1 = f1_score(label_errors_target, label_issues)\n",
    "            acc = accuracy_score(label_errors_target, label_issues)\n",
    "\n",
    "            result = {\n",
    "                \"dataset\": dataset,\n",
    "                \"model\": selected_model,\n",
    "                \"filter_by\": filter_by,\n",
    "                \"precision\": precision,\n",
    "                \"recall\": recall,\n",
    "                \"f1\": f1,\n",
    "                \"accuracy\": acc,\n",
    "                \"num_est_label_issues\": label_issues.sum(),\n",
    "                \"dataset_num_samples\": labels.shape[0],\n",
    "                \"dataset_num_label_errors\": label_errors_target.sum(),\n",
    "            }\n",
    "\n",
    "            results.append(result)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e42da170-3a34-422f-849c-9d0195df7568",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_filter_by_eval = pd.DataFrame(results)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75b7f272-87d0-4d7b-8b50-6a785c92cecd",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_filter_by_eval\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da6291d7-c103-495b-a88d-9907af4e89d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_filter_by_eval_f1 = pd.pivot_table(\n",
    "    df_filter_by_eval, values=\"f1\", index=[\"filter_by\"], columns=[\"dataset\"]\n",
    ").reset_index()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8264a4b1-90a2-44a3-ae0d-a2c1d220a75a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_filter_by_eval_f1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a329d71f-80a3-4948-8dd4-c74662b0f799",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1005f46-fdf1-4387-b5d1-06dd238773fc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "23829f9f-b56e-4aa7-8b58-42f04c0ae69b",
   "metadata": {},
   "source": [
    "## Generate Precision-Recall curve\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f1fef64a-5d86-4b55-8458-34a54fd684ea",
   "metadata": {},
   "source": [
    "**Note:** we can refactor the code to make it more concise but for now we use a for-loop and only plot for certain methods\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "546d6b5d-b380-4823-afa2-c9c2ef7e7b88",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams[\"figure.figsize\"] = (15, 10)\n",
    "\n",
    "df_list = []\n",
    "\n",
    "for data in precision_recall_curves:\n",
    "\n",
    "    # get data needed to plot precision-recall curve\n",
    "    method = data[\"method\"]\n",
    "    adjust_pred_probs = data[\"adjust_pred_probs\"]\n",
    "    dataset = data[\"dataset\"]\n",
    "    model = data[\"model\"]\n",
    "    label_quality_scores = data[\"label_quality_scores\"]\n",
    "    precision = data[\"precision\"]\n",
    "    recall = data[\"recall\"]\n",
    "    thresholds = data[\"thresholds\"]\n",
    "\n",
    "    # save to DataFrame\n",
    "    # ignore last precision, recall value because it's always 1, 0 respectively with no corresponding threshold\n",
    "    # https://stackoverflow.com/questions/31639016/in-scikits-precision-recall-curve-why-does-thresholds-have-a-different-dimensi\n",
    "    df_temp = pd.DataFrame(\n",
    "        {\n",
    "            \"precision\": precision[:-1],\n",
    "            \"recall\": recall[:-1],\n",
    "            \"thresholds\": thresholds,\n",
    "            \"model\": model,\n",
    "            \"method\": method,\n",
    "            \"adjust_pred_probs\": adjust_pred_probs,\n",
    "        }\n",
    "    )\n",
    "\n",
    "    df_list.append(df_temp)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8a20e75-7a7b-4f7f-a5c8-23fb55e8e583",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_temp[\"precision\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3106888-44ba-4312-912d-9de2ea089d79",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51c54e51-d11f-40e5-866a-e5f058323e79",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ffbedb47-30e4-4b30-ba10-8f27883a0ee5",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "plt.rcParams[\"figure.figsize\"] = (15, 10)\n",
    "\n",
    "score_params = \\\n",
    "[\n",
    "    # (\"self_confidence\", False),\n",
    "    (\"self_confidence\", True),\n",
    "    # (\"normalized_margin\", False),\n",
    "    (\"normalized_margin\", True),\n",
    "    (\"confidence_weighted_entropy\", False)\n",
    "]\n",
    "\n",
    "df_list = []\n",
    "\n",
    "for score_param in score_params:\n",
    "\n",
    "    method, adjust_pred_probs = score_param\n",
    "    \n",
    "    print(f\"Scoring label quality...\")\n",
    "    print(f\"  method: {method}\")\n",
    "    print(f\"  adjust_pred_probs: {adjust_pred_probs}\")\n",
    "\n",
    "    label_quality_scores = get_label_quality_scores(labels, pred_probs, method=method, adjust_pred_probs=adjust_pred_probs)\n",
    "\n",
    "    # compute accuracy of detecting label errors\n",
    "    auroc = roc_auc_score(label_errors_target, 1 - label_quality_scores)\n",
    "\n",
    "    # compute precision-recall curve\n",
    "    precision, recall, thresholds = precision_recall_curve(label_errors_target, 1 - label_quality_scores)\n",
    "\n",
    "    # save to DataFrame\n",
    "    # ignore last precision, recall value because it's always 1, 0 respectively with no corresponding threshold\n",
    "    # https://stackoverflow.com/questions/31639016/in-scikits-precision-recall-curve-why-does-thresholds-have-a-different-dimensi\n",
    "    df_temp = pd.DataFrame({\n",
    "        \"precision\": precision[:-1],\n",
    "        \"recall\": recall[:-1],\n",
    "        \"thresholds\": thresholds,\n",
    "        \"model\": model,\n",
    "        \"method\": method,\n",
    "        \"adjust_pred_probs\": adjust_pred_probs\n",
    "    })\n",
    "\n",
    "    df_list.append(df_temp)\n",
    "    \n",
    "    # plot\n",
    "    plt.plot(recall, precision, label=f\"{method}-{str(adjust_pred_probs)}\")\n",
    "    \n",
    "# combine DataFrames\n",
    "# df_all = pd.concat(df_list)\n",
    "\n",
    "# plot single dot (precision, recall) for each filter_by option\n",
    "for index, row in df_filter_by.iterrows():\n",
    "    filter_by = row[\"filter_by\"]\n",
    "    precision = row[\"precision\"]\n",
    "    recall = row[\"recall\"]\n",
    "    plt.plot(recall, precision, marker=\"o\", markersize=10, label=filter_by)\n",
    "\n",
    "plt.xlabel(\"Recall\", fontsize=14)\n",
    "plt.ylabel(\"Precision\", fontsize=14)\n",
    "plt.title(\"Precision-Recall Curve: Label Error Detection on CIFAR-10N-Worst \\n Model: swin_base_patch4_window7_224\", fontsize=20, fontweight=\"bold\")\n",
    "# plt.suptitle(\"\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}