experiments/uncovering_regions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b6939d9-9a04-4ea7-83f2-f2989fd9ff9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../src')\n",
    "import logging\n",
    "logging.getLogger().setLevel(logging.INFO)\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import torchvision.datasets as dset\n",
    "import torchvision.transforms as transforms\n",
    "import torch\n",
    "from tqdm import tqdm\n",
    "from sklearn.metrics import classification_report\n",
    "import pandas as pd\n",
    "from sklearn.metrics.pairwise import rbf_kernel\n",
    "from PIL import Image\n",
    "import requests\n",
    "import pickle\n",
    "from transformers import CLIPProcessor, CLIPModel\n",
    "import math\n",
    "print(torch.cuda.is_available())\n",
    "import pickle\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import sys\n",
    "import os\n",
    "import matplotlib\n",
    "from collections import Counter\n",
    "import datetime\n",
    "import torch\n",
    "import time\n",
    "# local imports\n",
    "from teacher_methods.teacher_domino import *\n",
    "from teacher_methods.teacher_kmeans import *\n",
    "from teacher_methods.teacher_selection import *\n",
    "from teacher_methods.teacher_gen import *\n",
    "from datasets_hai.bdd import *\n",
    "from datasets_hai.coco import *\n",
    "from datasets_hai.mmlu import *\n",
    "from datasets_hai.dynasent import *\n",
    "from datasets_hai.gaussians import *\n",
    "from utils.utils import *\n",
    "from utils.metrics_hai import *\n",
    "from utils.metrics_regions import *\n",
    "from synth_regions import *\n",
    "\n",
    "\n",
    "metrics_print = ['adjusted_rand_score', 'fowlkes_mallows_score']\n",
    "def get_metrics_region_process(data_runs):\n",
    "    results = {}\n",
    "    for metric in metrics_print:\n",
    "        list_metric = []\n",
    "        for j in range(len(data_runs[0])):\n",
    "            mean = np.mean([data_runs[i][j][metric] for i in range(len(data_runs))])\n",
    "            std = np.std([data_runs[i][j][metric] for i in range(len(data_runs))])/np.sqrt(len(data_runs))\n",
    "            list_metric.append((mean,std))\n",
    "        results[metric] = list_metric\n",
    "    return results\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ca32ecf-5b3f-414f-ba9e-010a7d01d649",
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha = 0.8\n",
    "beta_high = 0.2\n",
    "beta_low = 0.01\n",
    "delta = 2\n",
    "randomized_sampling = 1\n",
    "parallel_processes = 1\n",
    "kernel = rbf_kernel # K(.,.)\n",
    "metric_y = loss_01\n",
    "initialization_epochs = 200\n",
    "initialization_restarts = 50\n",
    "TOTAL_TRIALS = 5\n",
    "DATA_SIZES = [8]\n",
    "epochs = 2000\n",
    "lr = 0.001\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1129ea67-9a6f-4b09-90b3-51d35edb7cd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata_dimensions = 1\n",
    "GOOD_REGION_ACC = 0.95\n",
    "BAD_REGION_ACC = 0.6\n",
    "OTHER_REGION_ACC = 0.75\n",
    "N_CLAUSES = random.randint(4, 4) # number of clauses in each region\n",
    "CLAUSE_LENGTH = random.randint(1,1) # number of metadata dimensions in each clause\n",
    "\n",
    "MIN_REGION_SIZE = 0.01 # minimum size of a region\n",
    "MAX_REGION_SIZE = 0.2 # maximum size of a region\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d36e7717-71d4-4324-8198-bb04fe7b4f89",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# BDD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de2571ae-0c61-4639-8708-63db52d8a76e",
   "metadata": {},
   "outputs": [],
   "source": [
    "date_now = datetime.datetime.now()\n",
    "date_now = date_now.strftime(\"%Y-%m-%d_%H%M%S\")\n",
    "dataset = pickle.load(open(\"../data/cleaned_pkl/bdd_dataset.pkl\",\"rb\"))\n",
    "metadata_dimensions = len(dataset.metadata[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9491d8e0-807a-4924-846e-74a23a837373",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "# split data into 10% train and 90% test, get ids\n",
    "data_size_n = len(dataset.data_y)\n",
    "\n",
    "\n",
    "\n",
    "metrics_gen_region = []\n",
    "metrics_domino_region = []\n",
    "metrics_kmeans_region = []\n",
    "metrics_selec_region = []\n",
    "\n",
    "metrics_gen_test = []\n",
    "metrics_domino_test = []\n",
    "metrics_kmeans_test = []\n",
    "metrics_selec_test = []\n",
    "\n",
    "metrics_human_alone = []\n",
    "metrics_ai_alone = []\n",
    "\n",
    "max_teaching_points = max(DATA_SIZES)\n",
    "region_names = []\n",
    "for trial in range(TOTAL_TRIALS):\n",
    "    data_ids = list(range(0,len(dataset.data_y) ))\n",
    "    data_train_ids, data_test_ids = train_test_split(data_ids, test_size=0.3)\n",
    "\n",
    "    \n",
    "    ai_preds, hum_preds, true_regions, ai_regions_names, hum_regions_names = get_synth_ai_human_regions(dataset, metadata_dimensions, GOOD_REGION_ACC, BAD_REGION_ACC, OTHER_REGION_ACC, N_CLAUSES, CLAUSE_LENGTH, MIN_REGION_SIZE, MAX_REGION_SIZE)\n",
    "\n",
    "    print(ai_regions_names)\n",
    "    print(hum_regions_names)\n",
    "    region_names.append([ai_regions_names,hum_regions_names])\n",
    "    dataset.hum_preds = hum_preds\n",
    "    dataset.ai_preds = ai_preds\n",
    "    prior_rejector_preds = np.array([np.random.choice([0,1], p=[0.5,0.5]) for i in range(len(dataset.data_y))])\n",
    "\n",
    "\n",
    "    teacher_gen = TeacherGenerative(dataset.data_x[data_train_ids], dataset.data_y[data_train_ids], dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids],\n",
    "                                    prior_rejector_preds[data_train_ids], metric_y, max_teaching_points, alpha, beta_high, beta_low, delta)\n",
    "    teacher_gen.epochs = epochs\n",
    "    teacher_gen.lr = lr\n",
    "    teacher_gen.initialization_restarts = initialization_restarts\n",
    "    teacher_gen.initialization_epochs = initialization_epochs\n",
    "    teacher_gen.fit()\n",
    "\n",
    "\n",
    "\n",
    "    teacher_selection = TeacherSelective(dataset.data_x[data_train_ids], dataset.data_y[data_train_ids], dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids], prior_rejector_preds[data_train_ids], kernel, metric_y, max_teaching_points, alpha, beta_high, beta_low,  randomized_sampling, delta, parallel_processes)\n",
    "    print(\"SELECTION\")\n",
    "    teacher_selection.randomized_sampling = randomized_sampling\n",
    "    teacher_selection.parallel_processes = parallel_processes\n",
    "    teacher_selection.fit()\n",
    "\n",
    "    saved_selec_points = copy.deepcopy(teacher_selection.teaching_set)\n",
    "    saved_gen_points = copy.deepcopy(teacher_gen.teaching_set)\n",
    "    # fit generative and selection\n",
    "    metric_gen_region_trial = []\n",
    "    metric_selec_region_trial = []\n",
    "    metric_gen_test_trial = []\n",
    "    metric_selec_test_trial = []\n",
    "    metric_domino_region_trial = []\n",
    "    metric_domino_test_trial = []\n",
    "    metric_kmeans_region_trial = []\n",
    "    metric_kmeans_test_trial = []\n",
    "    metrics_human_alone_trial = []\n",
    "    metrics_ai_alone_trial = []\n",
    "\n",
    "    for data_size in DATA_SIZES:\n",
    "        try:\n",
    "            teacher_domino = TeacherDomino(dataset.data_x[data_train_ids], dataset.data_y[data_train_ids], dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids], dataset.ai_scores[data_train_ids], metric_y, n_pca_components = None, n_mixture_components = data_size , teaching_points = data_size)\n",
    "            teacher_domino.fit()\n",
    "        except:\n",
    "            print(\"Domino failed\")\n",
    "            teacher_domino = TeacherDomino(dataset.data_x[data_train_ids], dataset.data_y[data_train_ids], dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids], dataset.ai_scores[data_train_ids], metric_y, n_pca_components = None, n_mixture_components = 50 , teaching_points = data_size)\n",
    "            teacher_domino.fit()    \n",
    "            \n",
    "        teacher_kmeans = TeacherKmeans(dataset.data_x[data_train_ids], dataset.data_y[data_train_ids], dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids], metric_y, data_size)\n",
    "        print(\"Domino\")\n",
    "        teacher_domino.fit()\n",
    "        teacher_kmeans.fit()\n",
    "        teacher_selection.teaching_set = saved_selec_points[:data_size]\n",
    "        teacher_gen.teaching_set = saved_gen_points[:data_size]\n",
    "        # get eval results\n",
    "\n",
    "        human_only = compute_metrics( dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids], np.zeros(len(data_train_ids)), dataset.data_y[data_train_ids], metric_y)\n",
    "        ai_only = compute_metrics( dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids], np.ones(len(data_train_ids)), dataset.data_y[data_train_ids], metric_y)\n",
    "\n",
    "        metrics_human_alone_trial.append(human_only)\n",
    "        metrics_ai_alone_trial.append(ai_only)\n",
    "        # TEST, same code but use data_test_ids\n",
    "        domino_defer_preds = teacher_domino.get_defer_preds(dataset.data_x[data_test_ids])\n",
    "        domino_loss = compute_metrics( dataset.hum_preds[data_test_ids], dataset.ai_preds[data_test_ids], domino_defer_preds, dataset.data_y[data_test_ids], metric_y)\n",
    "        kmeans_defer_preds = teacher_kmeans.get_defer_preds(dataset.data_x[data_test_ids])\n",
    "        kmeans_loss = compute_metrics( dataset.hum_preds[data_test_ids], dataset.ai_preds[data_test_ids], kmeans_defer_preds, dataset.data_y[data_test_ids], metric_y)\n",
    "        selection_defer_preds = teacher_selection.get_defer_preds(dataset.data_x[data_test_ids], prior_rejector_preds[data_test_ids])\n",
    "        selection_loss = compute_metrics( dataset.hum_preds[data_test_ids], dataset.ai_preds[data_test_ids], selection_defer_preds, dataset.data_y[data_test_ids], metric_y)\n",
    "        gen_defer_preds = teacher_gen.get_defer_preds(dataset.data_x[data_test_ids], prior_rejector_preds[data_test_ids])\n",
    "        gen_loss = compute_metrics( dataset.hum_preds[data_test_ids], dataset.ai_preds[data_test_ids], gen_defer_preds, dataset.data_y[data_test_ids], metric_y)\n",
    "        human_only = compute_metrics( dataset.hum_preds[data_test_ids], dataset.ai_preds[data_test_ids], np.zeros(len(data_test_ids)), dataset.data_y[data_test_ids], metric_y)\n",
    "        ai_only = compute_metrics( dataset.hum_preds[data_test_ids], dataset.ai_preds[data_test_ids], np.ones(len(data_test_ids)), dataset.data_y[data_test_ids], metric_y)\n",
    "        metric_gen_test_trial.append(gen_loss)\n",
    "        metric_selec_test_trial.append(selection_loss)\n",
    "        metric_domino_test_trial.append(domino_loss)\n",
    "        metric_kmeans_test_trial.append(kmeans_loss)\n",
    "        # now get region preds on train\n",
    "        gen_region_labels = teacher_gen.get_region_labels(dataset.data_x[data_train_ids], prior_rejector_preds[data_train_ids])\n",
    "        selec_region_labels = teacher_selection.get_region_labels(dataset.data_x[data_train_ids], prior_rejector_preds[data_train_ids])\n",
    "        domino_region_labels = teacher_domino.get_region_labels(dataset.data_x[data_train_ids])\n",
    "        kmeans_region_labels = teacher_kmeans.get_region_labels(dataset.data_x[data_train_ids])\n",
    "        gen_region_metrics = get_region_metrics(true_regions[data_train_ids], gen_region_labels)\n",
    "        selec_region_metrics = get_region_metrics(true_regions[data_train_ids], selec_region_labels)\n",
    "        domino_region_metrics = get_region_metrics(true_regions[data_train_ids], domino_region_labels)\n",
    "        kmeans_region_metrics = get_region_metrics(true_regions[data_train_ids], kmeans_region_labels)\n",
    "        metric_gen_region_trial.append(gen_region_metrics)\n",
    "        metric_selec_region_trial.append(selec_region_metrics)\n",
    "        metric_domino_region_trial.append(domino_region_metrics)\n",
    "        metric_kmeans_region_trial.append(kmeans_region_metrics)\n",
    "\n",
    "        \n",
    "    metrics_human_alone.append(metrics_human_alone_trial)\n",
    "    metrics_ai_alone.append(metrics_ai_alone_trial)\n",
    "    metrics_gen_region.append(metric_gen_region_trial)\n",
    "    metrics_selec_region.append(metric_selec_region_trial)\n",
    "    metrics_domino_region.append(metric_domino_region_trial)\n",
    "    metrics_kmeans_region.append(metric_kmeans_region_trial)\n",
    "    metrics_gen_test.append(metric_gen_test_trial)\n",
    "    metrics_selec_test.append(metric_selec_test_trial)\n",
    "    metrics_domino_test.append(metric_domino_test_trial)\n",
    "    metrics_kmeans_test.append(metric_kmeans_test_trial)\n",
    "    data_save = {}\n",
    "    data_save[\"metrics_human_alone\"] = metrics_human_alone\n",
    "    data_save[\"metrics_ai_alone\"] = metrics_ai_alone\n",
    "    data_save[\"metrics_gen_region\"] = metrics_gen_region\n",
    "    data_save[\"metrics_selec_region\"] = metrics_selec_region\n",
    "    data_save[\"metrics_domino_region\"] = metrics_domino_region\n",
    "    data_save[\"metrics_kmeans_region\"] = metrics_kmeans_region\n",
    "    data_save[\"metrics_gen_test\"] = metrics_gen_test\n",
    "    data_save[\"metrics_selec_test\"] = metrics_selec_test\n",
    "    data_save[\"metrics_domino_test\"] = metrics_domino_test\n",
    "    data_save[\"metrics_kmeans_test\"] = metrics_kmeans_test\n",
    "    data_save[\"DATA_SIZES\"] = DATA_SIZES\n",
    "    data_save[\"region_names\"] = region_names\n",
    "    data_save[\"TOTAL_TRIALS\"] = TOTAL_TRIALS\n",
    "    with open(\"../exp_data/results/bdd_synth_\" + date_now + \".pkl\", \"wb\") as f:\n",
    "        pickle.dump(data_save, f)    \n",
    "    print(\"trial\", trial, \"done\")\n",
    "         \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24a87a0c-a662-4daf-abbd-f773b2990d5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "print(\"GEN\")\n",
    "print(get_metrics_region_process(metrics_gen_region))\n",
    "print(\"SELEC\")\n",
    "print(get_metrics_region_process(metrics_selec_region))\n",
    "print(\"DOMINO\")\n",
    "print(get_metrics_region_process(metrics_domino_region))\n",
    "print(\"KMEANS\")\n",
    "print(get_metrics_region_process(metrics_kmeans_region))\n",
    "alpha_overlap = 0.5\n",
    "\n",
    "# printing from a pickle file\n",
    "max_trials = 1# TOTAL_TRIALS\n",
    "ns = DATA_SIZES\n",
    "\n",
    "\n",
    "avgs_rand = [np.average([metrics_gen_test[triall][i][1]['score']   for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "stds_rand = [np.std([metrics_gen_test[triall][i][1]['score']     for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "plt.errorbar(ns,  avgs_rand, yerr=stds_rand, marker = \"d\",  label=f'GEN (ours)', alpha =alpha_overlap, color = \"black\")\n",
    "\n",
    "avgs_rand = [np.average([metrics_domino_test[triall][i][1]['score']   for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "stds_rand = [np.std([metrics_domino_test[triall][i][1]['score']  for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "plt.errorbar(ns,  avgs_rand, yerr=stds_rand, marker = \"o\",  label=f'DOMINO', alpha =alpha_overlap, color = \"red\")\n",
    "\n",
    "avgs_rand = [np.average([metrics_kmeans_test[triall][i][1]['score']   for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "stds_rand = [np.std([metrics_kmeans_test[triall][i][1]['score']  for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "plt.errorbar(ns,  avgs_rand, yerr=stds_rand, marker = \"x\",  label=f'KMEANS', alpha =alpha_overlap, color = \"blue\")\n",
    "\n",
    "\n",
    "avgs_rand = [np.average([metrics_selec_test[triall][i][1]['score']     for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "stds_rand = [np.std([metrics_selec_test[triall][i][1]['score']     for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "plt.errorbar(ns,  avgs_rand, yerr=stds_rand, marker = \"<\",  label=f'SELECT', alpha =alpha_overlap, color = \"green\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "ax = plt.gca()\n",
    "ax.spines['right'].set_visible(False)\n",
    "ax.spines['top'].set_visible(False)\n",
    "ax.get_xaxis().tick_bottom()    \n",
    "ax.get_yaxis().tick_left()   \n",
    "plt.grid()\n",
    "plt.legend(fontsize='xx-large')\n",
    "plt.legend()\n",
    "plt.ylabel('Test Loss ', fontsize='xx-large')\n",
    "plt.xlabel('Training data size', fontsize='xx-large')\n",
    "fig_size = plt.rcParams[\"figure.figsize\"]\n",
    "fig_size[0] = 6\n",
    "fig_size[1] = 4.2\n",
    "\n",
    "#plt.savefig(\"../exp_data/plots/plot_synth_data_realizable_\"+ date_now+\".pdf\", dpi = 1000, bbox_inches='tight')\n",
    "#plt.show()\n",
    "# "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de956a58-a545-47b2-92f3-4a116d9ab120",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# MMLU"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3dd293c2-5085-4148-a525-9149e64cefec",
   "metadata": {},
   "source": [
    "TODO: fix synth_regions predictions to be random "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e3a0ecf-1c31-4463-a085-653558b9c2f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "date_now = datetime.datetime.now()\n",
    "date_now = date_now.strftime(\"%Y-%m-%d_%H%M%S\")\n",
    "dataset =  pickle.load(open('../data/cleaned_pkl/mmlu_dataset.pkl',\"rb\"))\n",
    "metadata_dimensions = len(dataset.metadata[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "873c27cb-cdcb-430b-b49e-21b7aedae6fc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "# split data into 10% train and 90% test, get ids\n",
    "data_size_n = len(dataset.data_y)\n",
    "\n",
    "\n",
    "metrics_gen_region = []\n",
    "metrics_domino_region = []\n",
    "metrics_kmeans_region = []\n",
    "metrics_selec_region = []\n",
    "\n",
    "metrics_gen_test = []\n",
    "metrics_domino_test = []\n",
    "metrics_kmeans_test = []\n",
    "metrics_selec_test = []\n",
    "\n",
    "metrics_human_alone = []\n",
    "metrics_ai_alone = []\n",
    "\n",
    "max_teaching_points = max(DATA_SIZES)\n",
    "region_names = []\n",
    "for trial in range(TOTAL_TRIALS):\n",
    "    data_ids = list(range(0,len(dataset.data_y) ))\n",
    "    data_train_ids, data_test_ids = train_test_split(data_ids, test_size=0.3)\n",
    "\n",
    "    \n",
    "    ai_preds, hum_preds, true_regions, ai_regions_names, hum_regions_names = get_synth_ai_human_regions(dataset, metadata_dimensions, GOOD_REGION_ACC, BAD_REGION_ACC, OTHER_REGION_ACC, N_CLAUSES, CLAUSE_LENGTH, MIN_REGION_SIZE, MAX_REGION_SIZE)\n",
    "\n",
    "    print(ai_regions_names)\n",
    "    print(hum_regions_names)\n",
    "    region_names.append([ai_regions_names,hum_regions_names])\n",
    "    dataset.hum_preds = hum_preds\n",
    "    dataset.ai_preds = ai_preds\n",
    "    prior_rejector_preds = np.array([np.random.choice([0,1], p=[0.5,0.5]) for i in range(len(dataset.data_y))])\n",
    "\n",
    "\n",
    "    teacher_gen = TeacherGenerative(dataset.data_x[data_train_ids], dataset.data_y[data_train_ids], dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids],\n",
    "                                    prior_rejector_preds[data_train_ids], metric_y, max_teaching_points, alpha, beta_high, beta_low, delta)\n",
    "    teacher_gen.epochs = epochs\n",
    "    teacher_gen.lr = lr\n",
    "    teacher_gen.initialization_restarts = initialization_restarts\n",
    "    teacher_gen.initialization_epochs = initialization_epochs\n",
    "    teacher_gen.fit()\n",
    "\n",
    "\n",
    "\n",
    "    teacher_selection = TeacherSelective(dataset.data_x[data_train_ids], dataset.data_y[data_train_ids], dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids], prior_rejector_preds[data_train_ids], kernel, metric_y, max_teaching_points, alpha, beta_high, beta_low,  randomized_sampling, delta, parallel_processes)\n",
    "    print(\"SELECTION\")\n",
    "    teacher_selection.randomized_sampling = randomized_sampling\n",
    "    teacher_selection.parallel_processes = parallel_processes\n",
    "    teacher_selection.fit()\n",
    "\n",
    "    saved_selec_points = copy.deepcopy(teacher_selection.teaching_set)\n",
    "    saved_gen_points = copy.deepcopy(teacher_gen.teaching_set)\n",
    "    # fit generative and selection\n",
    "    metric_gen_region_trial = []\n",
    "    metric_selec_region_trial = []\n",
    "    metric_gen_test_trial = []\n",
    "    metric_selec_test_trial = []\n",
    "    metric_domino_region_trial = []\n",
    "    metric_domino_test_trial = []\n",
    "    metric_kmeans_region_trial = []\n",
    "    metric_kmeans_test_trial = []\n",
    "    metrics_human_alone_trial = []\n",
    "    metrics_ai_alone_trial = []\n",
    "\n",
    "    for data_size in DATA_SIZES:\n",
    "        data_y_one_hot = np.zeros((len(data_train_ids), 4))\n",
    "        dataset.data_y = dataset.data_y.astype(int)\n",
    "        data_y_one_hot[np.arange(len(data_train_ids)), dataset.data_y[data_train_ids]] = 1\n",
    "        ai_preds_one_hot = np.zeros((len(data_train_ids), 4))\n",
    "        dataset.ai_preds = dataset.ai_preds.astype(int)\n",
    "        ai_preds_one_hot[np.arange(len(data_train_ids)), dataset.ai_preds[data_train_ids]] = 1\n",
    "        hum_preds_one_hot = np.zeros((len(data_train_ids), 4))\n",
    "        dataset.hum_preds = dataset.hum_preds.astype(int)\n",
    "        hum_preds_one_hot[np.arange(len(data_train_ids)), dataset.hum_preds[data_train_ids]] = 1\n",
    "        try:\n",
    "            teacher_domino = TeacherDomino(dataset.data_x[data_train_ids], data_y_one_hot,hum_preds_one_hot,ai_preds_one_hot, ai_preds_one_hot, metric_y, n_pca_components = None, n_mixture_components = data_size , teaching_points = data_size)\n",
    "            teacher_domino.fit()\n",
    "        except:\n",
    "            print(\"Domino failed\")\n",
    "            teacher_domino = TeacherDomino(dataset.data_x[data_train_ids], data_y_one_hot,hum_preds_one_hot,ai_preds_one_hot, ai_preds_one_hot, metric_y, n_pca_components = None, n_mixture_components = 50 , teaching_points = data_size)\n",
    "            teacher_domino.fit()     \n",
    "            \n",
    "        teacher_kmeans = TeacherKmeans(dataset.data_x[data_train_ids], dataset.data_y[data_train_ids], dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids], metric_y, data_size)\n",
    "        print(\"Domino\")\n",
    "        teacher_domino.fit()\n",
    "        teacher_kmeans.fit()\n",
    "        teacher_selection.teaching_set = saved_selec_points[:data_size]\n",
    "        teacher_gen.teaching_set = saved_gen_points[:data_size]\n",
    "        # get eval results\n",
    "\n",
    "        human_only = compute_metrics( dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids], np.zeros(len(data_train_ids)), dataset.data_y[data_train_ids], metric_y)\n",
    "        ai_only = compute_metrics( dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids], np.ones(len(data_train_ids)), dataset.data_y[data_train_ids], metric_y)\n",
    "\n",
    "        metrics_human_alone_trial.append(human_only)\n",
    "        metrics_ai_alone_trial.append(ai_only)\n",
    "        # TEST, same code but use data_test_ids\n",
    "        domino_defer_preds = teacher_domino.get_defer_preds(dataset.data_x[data_test_ids])\n",
    "        domino_loss = compute_metrics( dataset.hum_preds[data_test_ids], dataset.ai_preds[data_test_ids], domino_defer_preds, dataset.data_y[data_test_ids], metric_y)\n",
    "        kmeans_defer_preds = teacher_kmeans.get_defer_preds(dataset.data_x[data_test_ids])\n",
    "        kmeans_loss = compute_metrics( dataset.hum_preds[data_test_ids], dataset.ai_preds[data_test_ids], kmeans_defer_preds, dataset.data_y[data_test_ids], metric_y)\n",
    "        selection_defer_preds = teacher_selection.get_defer_preds(dataset.data_x[data_test_ids], prior_rejector_preds[data_test_ids])\n",
    "        selection_loss = compute_metrics( dataset.hum_preds[data_test_ids], dataset.ai_preds[data_test_ids], selection_defer_preds, dataset.data_y[data_test_ids], metric_y)\n",
    "        gen_defer_preds = teacher_gen.get_defer_preds(dataset.data_x[data_test_ids], prior_rejector_preds[data_test_ids])\n",
    "        gen_loss = compute_metrics( dataset.hum_preds[data_test_ids], dataset.ai_preds[data_test_ids], gen_defer_preds, dataset.data_y[data_test_ids], metric_y)\n",
    "        human_only = compute_metrics( dataset.hum_preds[data_test_ids], dataset.ai_preds[data_test_ids], np.zeros(len(data_test_ids)), dataset.data_y[data_test_ids], metric_y)\n",
    "        ai_only = compute_metrics( dataset.hum_preds[data_test_ids], dataset.ai_preds[data_test_ids], np.ones(len(data_test_ids)), dataset.data_y[data_test_ids], metric_y)\n",
    "        metric_gen_test_trial.append(gen_loss)\n",
    "        metric_selec_test_trial.append(selection_loss)\n",
    "        metric_domino_test_trial.append(domino_loss)\n",
    "        metric_kmeans_test_trial.append(kmeans_loss)\n",
    "        # now get region preds on train\n",
    "        gen_region_labels = teacher_gen.get_region_labels(dataset.data_x[data_train_ids], prior_rejector_preds[data_train_ids])\n",
    "        selec_region_labels = teacher_selection.get_region_labels(dataset.data_x[data_train_ids], prior_rejector_preds[data_train_ids])\n",
    "        domino_region_labels = teacher_domino.get_region_labels(dataset.data_x[data_train_ids])\n",
    "        kmeans_region_labels = teacher_kmeans.get_region_labels(dataset.data_x[data_train_ids])\n",
    "        gen_region_metrics = get_region_metrics(true_regions[data_train_ids], gen_region_labels)\n",
    "        selec_region_metrics = get_region_metrics(true_regions[data_train_ids], selec_region_labels)\n",
    "        domino_region_metrics = get_region_metrics(true_regions[data_train_ids], domino_region_labels)\n",
    "        kmeans_region_metrics = get_region_metrics(true_regions[data_train_ids], kmeans_region_labels)\n",
    "        metric_gen_region_trial.append(gen_region_metrics)\n",
    "        metric_selec_region_trial.append(selec_region_metrics)\n",
    "        metric_domino_region_trial.append(domino_region_metrics)\n",
    "        metric_kmeans_region_trial.append(kmeans_region_metrics)\n",
    "\n",
    "        \n",
    "    metrics_human_alone.append(metrics_human_alone_trial)\n",
    "    metrics_ai_alone.append(metrics_ai_alone_trial)\n",
    "    metrics_gen_region.append(metric_gen_region_trial)\n",
    "    metrics_selec_region.append(metric_selec_region_trial)\n",
    "    metrics_domino_region.append(metric_domino_region_trial)\n",
    "    metrics_kmeans_region.append(metric_kmeans_region_trial)\n",
    "    metrics_gen_test.append(metric_gen_test_trial)\n",
    "    metrics_selec_test.append(metric_selec_test_trial)\n",
    "    metrics_domino_test.append(metric_domino_test_trial)\n",
    "    metrics_kmeans_test.append(metric_kmeans_test_trial)\n",
    "    data_save = {}\n",
    "    data_save[\"metrics_human_alone\"] = metrics_human_alone\n",
    "    data_save[\"metrics_ai_alone\"] = metrics_ai_alone\n",
    "    data_save[\"metrics_gen_region\"] = metrics_gen_region\n",
    "    data_save[\"metrics_selec_region\"] = metrics_selec_region\n",
    "    data_save[\"metrics_domino_region\"] = metrics_domino_region\n",
    "    data_save[\"metrics_kmeans_region\"] = metrics_kmeans_region\n",
    "    data_save[\"metrics_gen_test\"] = metrics_gen_test\n",
    "    data_save[\"metrics_selec_test\"] = metrics_selec_test\n",
    "    data_save[\"metrics_domino_test\"] = metrics_domino_test\n",
    "    data_save[\"metrics_kmeans_test\"] = metrics_kmeans_test\n",
    "    data_save[\"DATA_SIZES\"] = DATA_SIZES\n",
    "    data_save[\"region_names\"] = region_names\n",
    "    data_save[\"TOTAL_TRIALS\"] = TOTAL_TRIALS\n",
    "    with open(\"../exp_data/results/mmlu_synth_\" + date_now + \".pkl\", \"wb\") as f:\n",
    "        pickle.dump(data_save, f)    \n",
    "    print(\"trial\", trial, \"done\")\n",
    "         \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "abb4bb59-b6f1-4997-88ee-f4f3e92128f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "print(\"GEN\")\n",
    "print(get_metrics_region_process(metrics_gen_region))\n",
    "print(\"SELEC\")\n",
    "print(get_metrics_region_process(metrics_selec_region))\n",
    "print(\"DOMINO\")\n",
    "print(get_metrics_region_process(metrics_domino_region))\n",
    "print(\"KMEANS\")\n",
    "print(get_metrics_region_process(metrics_kmeans_region))\n",
    "alpha_overlap = 0.5\n",
    "\n",
    "alpha_overlap = 0.5\n",
    "\n",
    "# printing from a pickle file\n",
    "max_trials = TOTAL_TRIALS\n",
    "ns = DATA_SIZES\n",
    "\n",
    "\n",
    "avgs_rand = [np.average([metrics_gen_test[triall][i][1]['score']   for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "stds_rand = [np.std([metrics_gen_test[triall][i][1]['score']     for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "plt.errorbar(ns,  avgs_rand, yerr=stds_rand, marker = \"d\",  label=f'GEN (ours)', alpha =alpha_overlap, color = \"black\")\n",
    "\n",
    "avgs_rand = [np.average([metrics_domino_test[triall][i][1]['score']   for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "stds_rand = [np.std([metrics_domino_test[triall][i][1]['score']  for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "plt.errorbar(ns,  avgs_rand, yerr=stds_rand, marker = \"o\",  label=f'DOMINO', alpha =alpha_overlap, color = \"red\")\n",
    "\n",
    "avgs_rand = [np.average([metrics_kmeans_test[triall][i][1]['score']   for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "stds_rand = [np.std([metrics_kmeans_test[triall][i][1]['score']  for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "plt.errorbar(ns,  avgs_rand, yerr=stds_rand, marker = \"x\",  label=f'KMEANS', alpha =alpha_overlap, color = \"blue\")\n",
    "\n",
    "\n",
    "avgs_rand = [np.average([metrics_selec_test[triall][i][1]['score']     for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "stds_rand = [np.std([metrics_selec_test[triall][i][1]['score']     for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "plt.errorbar(ns,  avgs_rand, yerr=stds_rand, marker = \"<\",  label=f'SELECT', alpha =alpha_overlap, color = \"green\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "ax = plt.gca()\n",
    "ax.spines['right'].set_visible(False)\n",
    "ax.spines['top'].set_visible(False)\n",
    "ax.get_xaxis().tick_bottom()    \n",
    "ax.get_yaxis().tick_left()   \n",
    "plt.grid()\n",
    "plt.legend(fontsize='xx-large')\n",
    "plt.legend()\n",
    "plt.ylabel('Test Loss ', fontsize='xx-large')\n",
    "plt.xlabel('Training data size', fontsize='xx-large')\n",
    "fig_size = plt.rcParams[\"figure.figsize\"]\n",
    "fig_size[0] = 6\n",
    "fig_size[1] = 4.2\n",
    "\n",
    "#plt.savefig(\"../exp_data/plots/plot_synth_data_realizable_\"+ date_now+\".pdf\", dpi = 1000, bbox_inches='tight')\n",
    "#plt.show()\n",
    "# "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1963fde5-ddda-4c75-89ab-b7b5d620c780",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# MS-COCO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0877caae-8bb0-4c0f-becb-2c9c55009ad4",
   "metadata": {},
   "outputs": [],
   "source": [
    "date_now = datetime.datetime.now()\n",
    "date_now = date_now.strftime(\"%Y-%m-%d_%H%M%S\")\n",
    "dataset =  pickle.load(open('../data/cleaned_pkl/coco_dataset.pkl',\"rb\"))\n",
    "metadata_dimensions = len(dataset.metadata[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f322149-4e62-463d-b3db-f8018dee3774",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# split data into 10% train and 90% test, get ids\n",
    "data_size_n = len(dataset.data_y)\n",
    "\n",
    "\n",
    "\n",
    "metrics_gen_region = []\n",
    "metrics_domino_region = []\n",
    "metrics_kmeans_region = []\n",
    "metrics_selec_region = []\n",
    "\n",
    "metrics_gen_test = []\n",
    "metrics_domino_test = []\n",
    "metrics_kmeans_test = []\n",
    "metrics_selec_test = []\n",
    "\n",
    "metrics_human_alone = []\n",
    "metrics_ai_alone = []\n",
    "\n",
    "max_teaching_points = max(DATA_SIZES)\n",
    "region_names = []\n",
    "for trial in range(TOTAL_TRIALS):\n",
    "    data_ids = list(range(0,len(dataset.data_y) ))\n",
    "    data_train_ids, data_test_ids = train_test_split(data_ids, test_size=0.3)\n",
    "\n",
    "    \n",
    "    ai_preds, hum_preds, true_regions, ai_regions_names, hum_regions_names = get_synth_ai_human_regions(dataset, metadata_dimensions, GOOD_REGION_ACC, BAD_REGION_ACC, OTHER_REGION_ACC, N_CLAUSES, CLAUSE_LENGTH, MIN_REGION_SIZE, MAX_REGION_SIZE)\n",
    "\n",
    "    print(ai_regions_names)\n",
    "    print(hum_regions_names)\n",
    "    region_names.append([ai_regions_names,hum_regions_names])\n",
    "    dataset.hum_preds = hum_preds\n",
    "    dataset.ai_preds = ai_preds\n",
    "    prior_rejector_preds = np.array([np.random.choice([0,1], p=[0.5,0.5]) for i in range(len(dataset.data_y))])\n",
    "\n",
    "\n",
    "    teacher_gen = TeacherGenerative(dataset.data_x[data_train_ids], dataset.data_y[data_train_ids], dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids],\n",
    "                                    prior_rejector_preds[data_train_ids], metric_y, max_teaching_points, alpha, beta_high, beta_low, delta)\n",
    "    teacher_gen.epochs = epochs\n",
    "    teacher_gen.lr = lr\n",
    "    teacher_gen.initialization_restarts = initialization_restarts\n",
    "    teacher_gen.initialization_epochs = initialization_epochs\n",
    "    teacher_gen.fit()\n",
    "\n",
    "\n",
    "\n",
    "    teacher_selection = TeacherSelective(dataset.data_x[data_train_ids], dataset.data_y[data_train_ids], dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids], prior_rejector_preds[data_train_ids], kernel, metric_y, max_teaching_points, alpha, beta_high, beta_low,  randomized_sampling, delta, parallel_processes)\n",
    "    print(\"SELECTION\")\n",
    "    teacher_selection.randomized_sampling = randomized_sampling\n",
    "    teacher_selection.parallel_processes = parallel_processes\n",
    "    teacher_selection.fit()\n",
    "\n",
    "    saved_selec_points = copy.deepcopy(teacher_selection.teaching_set)\n",
    "    saved_gen_points = copy.deepcopy(teacher_gen.teaching_set)\n",
    "    # fit generative and selection\n",
    "    metric_gen_region_trial = []\n",
    "    metric_selec_region_trial = []\n",
    "    metric_gen_test_trial = []\n",
    "    metric_selec_test_trial = []\n",
    "    metric_domino_region_trial = []\n",
    "    metric_domino_test_trial = []\n",
    "    metric_kmeans_region_trial = []\n",
    "    metric_kmeans_test_trial = []\n",
    "    metrics_human_alone_trial = []\n",
    "    metrics_ai_alone_trial = []\n",
    "\n",
    "    for data_size in DATA_SIZES:\n",
    "        try:\n",
    "            teacher_domino = TeacherDomino(dataset.data_x[data_train_ids], dataset.data_y[data_train_ids], dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids], dataset.ai_scores[data_train_ids], metric_y, n_pca_components = None, n_mixture_components = data_size , teaching_points = data_size)\n",
    "            teacher_domino.fit()\n",
    "        except:\n",
    "            print(\"Domino failed\")\n",
    "            teacher_domino = TeacherDomino(dataset.data_x[data_train_ids], dataset.data_y[data_train_ids], dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids], dataset.ai_scores[data_train_ids], metric_y, n_pca_components = None, n_mixture_components = 50 , teaching_points = data_size)\n",
    "            teacher_domino.fit()    \n",
    "            \n",
    "        teacher_kmeans = TeacherKmeans(dataset.data_x[data_train_ids], dataset.data_y[data_train_ids], dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids], metric_y, data_size)\n",
    "        print(\"Domino\")\n",
    "        teacher_domino.fit()\n",
    "        teacher_kmeans.fit()\n",
    "        teacher_selection.teaching_set = saved_selec_points[:data_size]\n",
    "        teacher_gen.teaching_set = saved_gen_points[:data_size]\n",
    "        # get eval results\n",
    "\n",
    "        human_only = compute_metrics( dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids], np.zeros(len(data_train_ids)), dataset.data_y[data_train_ids], metric_y)\n",
    "        ai_only = compute_metrics( dataset.hum_preds[data_train_ids], dataset.ai_preds[data_train_ids], np.ones(len(data_train_ids)), dataset.data_y[data_train_ids], metric_y)\n",
    "\n",
    "        metrics_human_alone_trial.append(human_only)\n",
    "        metrics_ai_alone_trial.append(ai_only)\n",
    "        # TEST, same code but use data_test_ids\n",
    "        domino_defer_preds = teacher_domino.get_defer_preds(dataset.data_x[data_test_ids])\n",
    "        domino_loss = compute_metrics( dataset.hum_preds[data_test_ids], dataset.ai_preds[data_test_ids], domino_defer_preds, dataset.data_y[data_test_ids], metric_y)\n",
    "        kmeans_defer_preds = teacher_kmeans.get_defer_preds(dataset.data_x[data_test_ids])\n",
    "        kmeans_loss = compute_metrics( dataset.hum_preds[data_test_ids], dataset.ai_preds[data_test_ids], kmeans_defer_preds, dataset.data_y[data_test_ids], metric_y)\n",
    "        selection_defer_preds = teacher_selection.get_defer_preds(dataset.data_x[data_test_ids], prior_rejector_preds[data_test_ids])\n",
    "        selection_loss = compute_metrics( dataset.hum_preds[data_test_ids], dataset.ai_preds[data_test_ids], selection_defer_preds, dataset.data_y[data_test_ids], metric_y)\n",
    "        gen_defer_preds = teacher_gen.get_defer_preds(dataset.data_x[data_test_ids], prior_rejector_preds[data_test_ids])\n",
    "        gen_loss = compute_metrics( dataset.hum_preds[data_test_ids], dataset.ai_preds[data_test_ids], gen_defer_preds, dataset.data_y[data_test_ids], metric_y)\n",
    "        human_only = compute_metrics( dataset.hum_preds[data_test_ids], dataset.ai_preds[data_test_ids], np.zeros(len(data_test_ids)), dataset.data_y[data_test_ids], metric_y)\n",
    "        ai_only = compute_metrics( dataset.hum_preds[data_test_ids], dataset.ai_preds[data_test_ids], np.ones(len(data_test_ids)), dataset.data_y[data_test_ids], metric_y)\n",
    "        metric_gen_test_trial.append(gen_loss)\n",
    "        metric_selec_test_trial.append(selection_loss)\n",
    "        metric_domino_test_trial.append(domino_loss)\n",
    "        metric_kmeans_test_trial.append(kmeans_loss)\n",
    "        # now get region preds on train\n",
    "        gen_region_labels = teacher_gen.get_region_labels(dataset.data_x[data_train_ids], prior_rejector_preds[data_train_ids])\n",
    "        selec_region_labels = teacher_selection.get_region_labels(dataset.data_x[data_train_ids], prior_rejector_preds[data_train_ids])\n",
    "        domino_region_labels = teacher_domino.get_region_labels(dataset.data_x[data_train_ids])\n",
    "        kmeans_region_labels = teacher_kmeans.get_region_labels(dataset.data_x[data_train_ids])\n",
    "        gen_region_metrics = get_region_metrics(true_regions[data_train_ids], gen_region_labels)\n",
    "        selec_region_metrics = get_region_metrics(true_regions[data_train_ids], selec_region_labels)\n",
    "        domino_region_metrics = get_region_metrics(true_regions[data_train_ids], domino_region_labels)\n",
    "        kmeans_region_metrics = get_region_metrics(true_regions[data_train_ids], kmeans_region_labels)\n",
    "        metric_gen_region_trial.append(gen_region_metrics)\n",
    "        metric_selec_region_trial.append(selec_region_metrics)\n",
    "        metric_domino_region_trial.append(domino_region_metrics)\n",
    "        metric_kmeans_region_trial.append(kmeans_region_metrics)\n",
    "\n",
    "        \n",
    "    metrics_human_alone.append(metrics_human_alone_trial)\n",
    "    metrics_ai_alone.append(metrics_ai_alone_trial)\n",
    "    metrics_gen_region.append(metric_gen_region_trial)\n",
    "    metrics_selec_region.append(metric_selec_region_trial)\n",
    "    metrics_domino_region.append(metric_domino_region_trial)\n",
    "    metrics_kmeans_region.append(metric_kmeans_region_trial)\n",
    "    metrics_gen_test.append(metric_gen_test_trial)\n",
    "    metrics_selec_test.append(metric_selec_test_trial)\n",
    "    metrics_domino_test.append(metric_domino_test_trial)\n",
    "    metrics_kmeans_test.append(metric_kmeans_test_trial)\n",
    "    data_save = {}\n",
    "    data_save[\"metrics_human_alone\"] = metrics_human_alone\n",
    "    data_save[\"metrics_ai_alone\"] = metrics_ai_alone\n",
    "    data_save[\"metrics_gen_region\"] = metrics_gen_region\n",
    "    data_save[\"metrics_selec_region\"] = metrics_selec_region\n",
    "    data_save[\"metrics_domino_region\"] = metrics_domino_region\n",
    "    data_save[\"metrics_kmeans_region\"] = metrics_kmeans_region\n",
    "    data_save[\"metrics_gen_test\"] = metrics_gen_test\n",
    "    data_save[\"metrics_selec_test\"] = metrics_selec_test\n",
    "    data_save[\"metrics_domino_test\"] = metrics_domino_test\n",
    "    data_save[\"metrics_kmeans_test\"] = metrics_kmeans_test\n",
    "    data_save[\"DATA_SIZES\"] = DATA_SIZES\n",
    "    data_save[\"region_names\"] = region_names\n",
    "    data_save[\"TOTAL_TRIALS\"] = TOTAL_TRIALS\n",
    "    with open(\"../exp_data/results/coco_synth_\" + date_now + \".pkl\", \"wb\") as f:\n",
    "        pickle.dump(data_save, f)    \n",
    "    print(\"trial\", trial, \"done\")\n",
    "         \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "064e4644-3f8a-41f1-ba28-2a6b38741165",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"GEN\")\n",
    "print(get_metrics_region_process(metrics_gen_region))\n",
    "print(\"SELEC\")\n",
    "print(get_metrics_region_process(metrics_selec_region))\n",
    "print(\"DOMINO\")\n",
    "print(get_metrics_region_process(metrics_domino_region))\n",
    "print(\"KMEANS\")\n",
    "print(get_metrics_region_process(metrics_kmeans_region))\n",
    "alpha_overlap = 0.5\n",
    "\n",
    "# printing from a pickle file\n",
    "max_trials = TOTAL_TRIALS\n",
    "ns = DATA_SIZES\n",
    "\n",
    "\n",
    "avgs_rand = [np.average([metrics_gen_test[triall][i][1]['score']   for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "stds_rand = [np.std([metrics_gen_test[triall][i][1]['score']     for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "plt.errorbar(ns,  avgs_rand, yerr=stds_rand, marker = \"d\",  label=f'GEN (ours)', alpha =alpha_overlap, color = \"black\")\n",
    "\n",
    "avgs_rand = [np.average([metrics_domino_test[triall][i][1]['score']   for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "stds_rand = [np.std([metrics_domino_test[triall][i][1]['score']  for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "plt.errorbar(ns,  avgs_rand, yerr=stds_rand, marker = \"o\",  label=f'DOMINO', alpha =alpha_overlap, color = \"red\")\n",
    "\n",
    "avgs_rand = [np.average([metrics_kmeans_test[triall][i][1]['score']   for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "stds_rand = [np.std([metrics_kmeans_test[triall][i][1]['score']  for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "plt.errorbar(ns,  avgs_rand, yerr=stds_rand, marker = \"x\",  label=f'KMEANS', alpha =alpha_overlap, color = \"blue\")\n",
    "\n",
    "\n",
    "avgs_rand = [np.average([metrics_selec_test[triall][i][1]['score']     for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "stds_rand = [np.std([metrics_selec_test[triall][i][1]['score']     for triall in range(max_trials)]) for i in range(len(ns))]\n",
    "plt.errorbar(ns,  avgs_rand, yerr=stds_rand, marker = \"<\",  label=f'SELECT', alpha =alpha_overlap, color = \"green\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "ax = plt.gca()\n",
    "ax.spines['right'].set_visible(False)\n",
    "ax.spines['top'].set_visible(False)\n",
    "ax.get_xaxis().tick_bottom()    \n",
    "ax.get_yaxis().tick_left()   \n",
    "plt.grid()\n",
    "plt.legend(fontsize='xx-large')\n",
    "plt.legend()\n",
    "plt.ylabel('Test Loss ', fontsize='xx-large')\n",
    "plt.xlabel('Training data size', fontsize='xx-large')\n",
    "fig_size = plt.rcParams[\"figure.figsize\"]\n",
    "fig_size[0] = 6\n",
    "fig_size[1] = 4.2\n",
    "\n",
    "#plt.savefig(\"../exp_data/plots/plot_synth_data_realizable_\"+ date_now+\".pdf\", dpi = 1000, bbox_inches='tight')\n",
    "#plt.show()\n",
    "# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "151a03df-f60a-4412-bb03-d8d0b35af9cf",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:teach_vision] *",
   "language": "python",
   "name": "conda-env-teach_vision-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}