Plotting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c093589c-7687-4ac0-865d-47d64d066f58",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "07b74355-d0b7-466b-8254-bfe5bf8892a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set_theme(style=\"whitegrid\", font_scale=3.)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e3afedf-f54b-44d0-afaf-493cff40a76c",
   "metadata": {},
   "source": [
    "### Val vs Test accuracy plot (Fig. 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bc0dbc25-73a7-45b3-b958-292ef17b503b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load saved data\n",
    "percent_acc = pd.read_csv('outputs/oracle_accuracy.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7f19d5a4-c8b6-4e13-a2d7-3dd2cfaced26",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/patara/anaconda3/envs/patara/lib/python3.9/site-packages/seaborn/relational.py:651: UserWarning: You passed a edgecolor/edgecolors ('w') for an unfilled marker ('+').  Matplotlib is ignoring the edgecolor in favor of the facecolor.  This behavior may change in the future.\n",
      "  points = ax.scatter(*args, **kws)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved\n"
     ]
    }
   ],
   "source": [
    "f, ax = plt.subplots(figsize=(9, 9))\n",
    "g = sns.scatterplot(data=percent_acc, x='validation', y='test', color='#0000FF', ax=ax, marker='+')\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Validation accuracy %', ylabel='Test accuracy %', xlim=[0, 100], ylim=[0, 100])\n",
    "f.savefig('figures/fig3a.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close(f)\n",
    "print('saved')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "47b9ec12-eef8-4a67-a2ee-4ccb7eb67799",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved\n"
     ]
    }
   ],
   "source": [
    "high_acc = percent_acc[percent_acc['validation'] >= 93.5].reset_index(drop=True)\n",
    "\n",
    "f, ax = plt.subplots(figsize=(9, 9))\n",
    "g = sns.scatterplot(data=high_acc, x='validation', y='test', color='#0000FF', ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Validation accuracy %', ylabel='Test accuracy %', xlim=[93.5, 95.5], ylim=[93.5, 95.5])\n",
    "f.savefig('figures/fig3b.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close(f)\n",
    "print('saved')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0011da31-9057-410b-a541-f0814b7fc3fa",
   "metadata": {},
   "source": [
    "### Performance Comparison Plot (Fig. 4) --- NAS-Bench-101"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "368ed677-8e1b-4269-a0d3-71413bb859b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set_theme(style=\"darkgrid\", font_scale=1.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0abb8ca8-fb91-4f7d-af0e-ad4a5177afa0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# number of samples vs validation accuracy\n",
    "random_val_avg_by_samples = np.load('outputs/random_val_avg_by_samples.npy')\n",
    "random_val_std_by_samples = np.load('outputs/random_val_std_by_samples.npy')\n",
    "\n",
    "np_val_avg_by_samples = np.load('outputs/np_val_avg_by_samples.npy')\n",
    "np_val_std_by_samples = np.load('outputs/np_val_std_by_samples.npy')\n",
    "\n",
    "np2_val_avg_by_samples = np.load('outputs/np2_val_avg_by_samples.npy')\n",
    "np2_val_std_by_samples = np.load('outputs/np2_val_std_by_samples.npy')\n",
    "\n",
    "oracle_val_acc = np.load('outputs/oracle_val_acc.npy')\n",
    "oracle_val_std = np.load('outputs/oracle_val_std.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "630c7931-1c62-487e-949d-601930ccf36c",
   "metadata": {},
   "outputs": [],
   "source": [
    "np_index = [i + 173 for i in range(len(np_val_avg_by_samples))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "73e8aa19-d274-4d8d-93f5-f518214c4327",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(9, 9))\n",
    "\n",
    "# axes settings\n",
    "ax.set(xlim = [0, 500], ylim = [86, 96], xlabel='The number of samples', ylabel='Validation accuracy %',)\n",
    "\n",
    "# neural predictor plot\n",
    "ax.vlines(x=172, ymin=0, ymax=100, colors='green', linestyle='dotted')\n",
    "\n",
    "plt.plot(np_index, np2_val_avg_by_samples, color='green', label='neural predictor w classifier')\n",
    "plt.fill_between(range(173, 501), y1=np2_val_avg_by_samples-np2_val_std_by_samples, y2=np2_val_avg_by_samples+np2_val_std_by_samples, facecolor='green', alpha=0.2)\n",
    "\n",
    "plt.plot(np_index, np_val_avg_by_samples, color='green', label='neural predictor w/o classifier', linestyle='dashed')\n",
    "plt.fill_between(range(173, 501), y1=np_val_avg_by_samples-np_val_std_by_samples, y2=np_val_avg_by_samples+np_val_std_by_samples, facecolor='green', alpha=0.2)\n",
    "\n",
    "\n",
    "# random search plot\n",
    "plt.plot(random_val_avg_by_samples, color='red', label='random search', linestyle='dashed')\n",
    "plt.fill_between(range(1, 501), y1=random_val_avg_by_samples-random_val_std_by_samples, y2=random_val_avg_by_samples+random_val_std_by_samples, facecolor='red', alpha=0.2)\n",
    "\n",
    "# oracle line\n",
    "ax.hlines(y=oracle_val_acc, xmin=0, xmax=500, color='black', label='oracle')\n",
    "ax.fill_between(range(1, 501), y1=oracle_val_acc-oracle_val_std, y2=oracle_val_acc+oracle_val_std, facecolor='black', alpha=0.2)\n",
    "\n",
    "# show plot with legend\n",
    "plt.legend(loc='lower right')\n",
    "plt.close()\n",
    "\n",
    "f.savefig('figures/fig4a.pdf', format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "99927960-87e1-496d-bd63-f95855501aaa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# number of samples vs test accuracy\n",
    "random_test_avg_by_samples = np.load('outputs/random_test_avg_by_samples.npy')\n",
    "random_test_std_by_samples = np.load('outputs/random_test_std_by_samples.npy')\n",
    "\n",
    "np_test_avg_by_samples = np.load('outputs/np_test_avg_by_samples.npy')\n",
    "np_test_std_by_samples = np.load('outputs/np_test_std_by_samples.npy')\n",
    "\n",
    "np2_test_avg_by_samples = np.load('outputs/np2_test_avg_by_samples.npy')\n",
    "np2_test_std_by_samples = np.load('outputs/np2_test_std_by_samples.npy')\n",
    "\n",
    "oracle_test_acc = np.load('outputs/oracle_test_acc.npy')\n",
    "oracle_test_std = np.load('outputs/oracle_test_std.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4c6f2c6a-9984-455d-b47d-1012f6ba1fcc",
   "metadata": {},
   "outputs": [],
   "source": [
    "np_index = [i + 173 for i in range(len(np_test_avg_by_samples))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8257582b-9616-4744-aa7b-26c0bcd6d5e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(9, 9))\n",
    "\n",
    "# axes settings\n",
    "ax.set(xlim = [0, 500], ylim = [85, 95], xlabel='The number of samples', ylabel='Test accuracy %',)\n",
    "\n",
    "# neural predictor plot\n",
    "ax.vlines(x=172, ymin=0, ymax=100, colors='green', linestyle='dotted')\n",
    "\n",
    "plt.plot(np_index, np2_test_avg_by_samples, color='green', label='neural predictor w classifier')\n",
    "plt.fill_between(range(173, 501), y1=np2_test_avg_by_samples-np2_test_std_by_samples, y2=np2_test_avg_by_samples+np2_test_std_by_samples, facecolor='green', alpha=0.2)\n",
    "\n",
    "plt.plot(np_index, np_test_avg_by_samples, color='green', label='neural predictor w/o classifier', linestyle='dashed')\n",
    "plt.fill_between(range(173, 501), y1=np_test_avg_by_samples-np_test_std_by_samples, y2=np_test_avg_by_samples+np_test_std_by_samples, facecolor='green', alpha=0.2)\n",
    "\n",
    "\n",
    "# random search plot\n",
    "plt.plot(random_test_avg_by_samples, color='red', label='random search', linestyle='dashed')\n",
    "plt.fill_between(range(1, 501), y1=random_test_avg_by_samples-random_test_std_by_samples, y2=random_test_avg_by_samples+random_test_std_by_samples, facecolor='red', alpha=0.2)\n",
    "\n",
    "# oracle line\n",
    "ax.hlines(y=oracle_test_acc, xmin=0, xmax=500, color='black', label='oracle')\n",
    "ax.fill_between(range(1, 501), y1=oracle_test_acc-oracle_test_std, y2=oracle_test_acc+oracle_test_std, facecolor='black', alpha=0.2)\n",
    "\n",
    "# show plot with legend\n",
    "plt.legend(loc='lower right')\n",
    "plt.close()\n",
    "\n",
    "f.savefig('figures/fig4b.pdf', format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ef08bdc6-e95a-4272-8fd4-beb4c1a1301c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# number of samples vs test accuracy\n",
    "random_test_avg = np.load('outputs/random_test_avg_by_time.npy')\n",
    "random_test_std = np.load('outputs/random_test_std_by_time.npy')\n",
    "\n",
    "np_test_avg = np.load('outputs/np_test_avg_by_time.npy')\n",
    "np_test_std = np.load('outputs/np_test_std_by_time.npy')\n",
    "\n",
    "np2_test_avg = np.load('outputs/np2_test_avg_by_time.npy')\n",
    "np2_test_std = np.load('outputs/np2_test_std_by_time.npy')\n",
    "\n",
    "oracle_test_acc = np.load('outputs/oracle_test_acc.npy')\n",
    "oracle_test_std = np.load('outputs/oracle_test_std.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b6f98249-5db0-4f92-a8db-a20e218eef83",
   "metadata": {},
   "outputs": [],
   "source": [
    "np_index = [i + 134 for i in range(len(np_test_avg))]\n",
    "np2_index = [i + 130 for i in range(len(np2_test_avg))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "10f538d7-0bf8-436e-9251-cdcbd0d4becd",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(9, 9))\n",
    "\n",
    "# axes settings\n",
    "ax.set(xlim = [0, 500], ylim = [90., 95.], xlabel='Total training time x 1600 (seconds)', ylabel='Test accuracy %')\n",
    "\n",
    "# neural predictor plot\n",
    "ax.vlines(x=131, ymin=0, ymax=100, colors='green', linestyle='dotted')\n",
    "\n",
    "plt.plot(np2_index, np2_test_avg, color='green', label='neural predictor w classifier')\n",
    "plt.fill_between(range(131, 501), y1=np2_test_avg-np2_test_std, y2=np2_test_avg+np2_test_std, facecolor='green', alpha=0.2)\n",
    "\n",
    "plt.plot(np_index, np_test_avg, color='green', label='neural predictor w/o classifier', linestyle='dashed')\n",
    "plt.fill_between(range(134, 501), y1=np_test_avg-np_test_std, y2=np_test_avg+np_test_std, facecolor='green', alpha=0.2)\n",
    "\n",
    "\n",
    "# random search plot\n",
    "plt.plot(random_test_avg, color='red', label='random search', linestyle='dashed')\n",
    "plt.fill_between(range(1, 501), y1=random_test_avg-random_test_std, y2=random_test_avg+random_test_std, facecolor='red', alpha=0.2)\n",
    "\n",
    "# oracle line\n",
    "ax.hlines(y=oracle_test_acc, xmin=0, xmax=500, color='black', label='oracle')\n",
    "ax.fill_between(range(1, 501), y1=oracle_test_acc-oracle_test_std, y2=oracle_test_acc+oracle_test_std, facecolor='black', alpha=0.2)\n",
    "\n",
    "# show plot with legend\n",
    "plt.legend(loc='lower right')\n",
    "plt.close()\n",
    "\n",
    "f.savefig('figures/fig4c.pdf', format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7083810c-a151-44ef-93c1-263d3a61beba",
   "metadata": {},
   "source": [
    "### Performance Comparison Plot (Fig. 4) -- NAS-Bench-NLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9b8eab67-e917-45a6-b732-f00ebedb9b06",
   "metadata": {},
   "outputs": [],
   "source": [
    "oracle_loss = np.load('outputs/nlp_oracle_loss.npy', allow_pickle=True).ravel()[0]\n",
    "oracle_std = np.load('outputs/nlp_oracle_std.npy', allow_pickle=True).ravel()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f23435d8-2d9a-423e-971a-3301661666ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "random_val_avg = np.load('outputs/nlp_random_val_avg_by_samples.npy')\n",
    "random_val_std = np.load('outputs/nlp_random_val_std_by_samples.npy')\n",
    "random_test_avg = np.load('outputs/nlp_random_test_avg_by_samples.npy')\n",
    "random_test_std = np.load('outputs/nlp_random_test_std_by_samples.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8ca2b806-767a-4a9e-9d15-767e473f329f",
   "metadata": {},
   "outputs": [],
   "source": [
    "np_val_avg = np.load('outputs/nlp_np_val_avg_by_samples.npy')\n",
    "np_val_std = np.load('outputs/nlp_np_val_std_by_samples.npy')\n",
    "np_test_avg = np.load('outputs/nlp_np_test_avg_by_samples.npy')\n",
    "np_test_std = np.load('outputs/nlp_np_test_std_by_samples.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ce54d82f-5f49-4c8b-ab87-617be1a22743",
   "metadata": {},
   "outputs": [],
   "source": [
    "np_index = [i + 173 for i in range(len(np_val_avg))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b012bd15-2ad6-4503-b55c-36c5c0be8bac",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(9, 9))\n",
    "\n",
    "# axes settings\n",
    "ax.set(xlim = [0, 500], ylim = [3.5, 6.], xlabel='The number of samples', ylabel='Validation loss',)\n",
    "\n",
    "# neural predictor plot\n",
    "ax.vlines(x=172, ymin=0, ymax=100, colors='green', linestyle='dotted')\n",
    "\n",
    "plt.plot(np_index, np_val_avg, color='green', label='neural predictor')\n",
    "plt.fill_between(range(173, 501), y1=np_val_avg-np_val_std, y2=np_val_avg+np_val_std, facecolor='green', alpha=0.2)\n",
    "\n",
    "# # random search plot\n",
    "plt.plot(random_val_avg, color='red', label='random search', linestyle='dashed')\n",
    "plt.fill_between(range(1, 501), y1=random_val_avg-random_val_std, y2=random_val_avg+random_val_std, facecolor='red', alpha=0.2)\n",
    "\n",
    "# oracle line\n",
    "ax.hlines(y=oracle_loss['val_loss'], xmin=0, xmax=500, color='black', label='oracle')\n",
    "ax.fill_between(range(1, 501), y1=oracle_loss['val_loss']-oracle_std['val_std'], y2=oracle_loss['val_loss']+oracle_std['val_std'], facecolor='black', alpha=0.2)\n",
    "\n",
    "# show plot with legend\n",
    "plt.legend(loc='lower right')\n",
    "plt.close()\n",
    "\n",
    "f.savefig('figures/fig4a-nlp.pdf', format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "82a86c22-2081-42d2-9e0f-cda305140054",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(9, 9))\n",
    "\n",
    "# axes settings\n",
    "ax.set(xlim = [0, 500], ylim = [3.5, 6.], xlabel='The number of samples', ylabel='Test loss',)\n",
    "\n",
    "# neural predictor plot\n",
    "ax.vlines(x=172, ymin=0, ymax=100, colors='green', linestyle='dotted')\n",
    "\n",
    "plt.plot(np_index, np_test_avg, color='green', label='neural predictor')\n",
    "plt.fill_between(range(173, 501), y1=np_test_avg-np_test_std, y2=np_test_avg+np_test_std, facecolor='green', alpha=0.2)\n",
    "\n",
    "# # random search plot\n",
    "plt.plot(random_test_avg, color='red', label='random search', linestyle='dashed')\n",
    "plt.fill_between(range(1, 501), y1=random_test_avg-random_test_std, y2=random_test_avg+random_test_std, facecolor='red', alpha=0.2)\n",
    "\n",
    "# oracle line\n",
    "ax.hlines(y=oracle_loss['test_loss'], xmin=0, xmax=500, color='black', label='oracle')\n",
    "ax.fill_between(range(1, 501), y1=oracle_loss['test_loss']-oracle_std['test_std'], y2=oracle_loss['test_loss']+oracle_std['test_std'], facecolor='black', alpha=0.2)\n",
    "\n",
    "# show plot with legend\n",
    "plt.legend(loc='lower right')\n",
    "plt.close()\n",
    "\n",
    "f.savefig('figures/fig4b-nlp.pdf', format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49e8f4c1-c44e-45d8-9ee4-a9f3b3cf9d94",
   "metadata": {},
   "source": [
    "### Classifier Filtering (Fig. 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "23c5613f-a33c-4215-a30d-322a448456ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load saved results\n",
    "train_clf_val_acc = np.load('outputs/train_clf_data_val_acc.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d62befc3-5a72-4455-953a-1d3ccf47c82f",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(9, 9))\n",
    "\n",
    "# axes settings\n",
    "ax.set(xlim = [0, 100], ylim=[0, 60], xlabel='Real validation accuracy %', ylabel='Counts')\n",
    "\n",
    "plt.hist(train_clf_val_acc, color='red', bins=10)\n",
    "plt.close()\n",
    "\n",
    "f.savefig('figures/fig6a.pdf', format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "254ae16b-0710-4ac5-a670-a1517665d651",
   "metadata": {},
   "outputs": [],
   "source": [
    "acc_set = np.load('outputs/clf_acc_set.npy')\n",
    "inacc_set = np.load('outputs/clf_inacc_set.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "323567c9-bbe6-4d15-a58d-b64bfa82a3d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(9, 9))\n",
    "\n",
    "# axes settings\n",
    "ax.set(xlim = [86, 95], ylim=[0, 14000], xlabel='Real validation accuracy %', ylabel='Counts')\n",
    "\n",
    "plt.hist(acc_set, color='green', histtype='step', bins=500, label='accurate', linewidth=2)\n",
    "plt.hist(inacc_set, color='red', histtype='step', bins=500, label='inaccurate', linewidth=2)\n",
    "\n",
    "# show plot with legend\n",
    "plt.legend(loc='upper left')\n",
    "plt.close()\n",
    "\n",
    "f.savefig('figures/fig6b.pdf', format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec2d1e4c-7ea9-4d5f-ac6e-cd3bd8375f6d",
   "metadata": {},
   "source": [
    "### Trade-off N vs K (Fig. 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "172d0934-613d-465a-b277-0a007e3942bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "oracle_val = np.load('outputs/oracle_val_acc.npy')\n",
    "oracle_test = np.load('outputs/oracle_test_acc.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "1bf1e86f-0522-4d69-914f-602439594439",
   "metadata": {},
   "outputs": [],
   "source": [
    "random_val = np.load('outputs/random_val_avg_2000.npy')\n",
    "random_test = np.load('outputs/random_test_avg_2000.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "be5edbfd-1056-4cb2-ada2-408307f9cbfc",
   "metadata": {},
   "outputs": [],
   "source": [
    "np_samples = np.load('outputs/ablation_study_by_samples.npy', allow_pickle=True).ravel()[0]\n",
    "np_time = np.load('outputs/ablation_study_by_time.npy', allow_pickle=True).ravel()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2baa11ba-c711-4726-add6-c814f9cdb167",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(9, 9))\n",
    "\n",
    "# axes settings\n",
    "ax.set(xlim = [0, 2000], ylim = [93, 95.4], xlabel='The number of samples', ylabel='Validation accuracy %',)\n",
    "\n",
    "# neural predictor plot\n",
    "ax.vlines(x=860, ymin=0, ymax=100, colors='pink', linestyle='dotted')\n",
    "plt.plot([i + 860 for i in range(len(np_samples[860]['np_val_avg']))], np_samples[860]['np_val_avg'], color='pink', label='neural predictor 860', linewidth=3)\n",
    "\n",
    "ax.vlines(x=344, ymin=0, ymax=100, colors='brown', linestyle='dotted')\n",
    "plt.plot([i + 344 for i in range(len(np_samples[344]['np_val_avg']))], np_samples[344]['np_val_avg'], color='brown', label='neural predictor 344', linewidth=3)\n",
    "\n",
    "ax.vlines(x=172, ymin=0, ymax=100, colors='green', linestyle='dotted')\n",
    "plt.plot([i + 172 for i in range(len(np_samples[172]['np_val_avg']))], np_samples[172]['np_val_avg'], color='green', label='neural predictor 172', linewidth=3)\n",
    "\n",
    "ax.vlines(x=129, ymin=0, ymax=100, colors='orange', linestyle='dotted')\n",
    "plt.plot([i + 129 for i in range(len(np_samples[129]['np_val_avg']))], np_samples[129]['np_val_avg'], color='orange', label='neural predictor 129', linewidth=3)\n",
    "\n",
    "ax.vlines(x=86, ymin=0, ymax=100, colors='purple', linestyle='dotted')\n",
    "plt.plot([i + 86 for i in range(len(np_samples[86]['np_val_avg']))], np_samples[86]['np_val_avg'], color='purple', label='neural predictor 86', linewidth=3)\n",
    "\n",
    "ax.vlines(x=43, ymin=0, ymax=100, colors='yellow', linestyle='dotted')\n",
    "plt.plot([i + 43 for i in range(len(np_samples[43]['np_val_avg']))], np_samples[43]['np_val_avg'], color='yellow', label='neural predictor 43', linewidth=3)\n",
    "\n",
    "\n",
    "# random search plot\n",
    "plt.plot(random_val, color='red', label='random search', linestyle='dashed', linewidth=3)\n",
    "\n",
    "# oracle line\n",
    "ax.hlines(y=oracle_val, xmin=0, xmax=2000, color='black', label='oracle', linewidth=3)\n",
    "\n",
    "# show plot with legend\n",
    "plt.legend(loc='lower right')\n",
    "plt.close()\n",
    "\n",
    "f.savefig('figures/fig7a.pdf', format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "70c058fa-5eaa-4589-964b-1a6e15b81f37",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(9, 9))\n",
    "\n",
    "# axes settings\n",
    "ax.set(xlim = [0, 2000], ylim = [92.75, 94.75], xlabel='The number of samples', ylabel='Test accuracy %',)\n",
    "\n",
    "# neural predictor plot\n",
    "ax.vlines(x=860, ymin=0, ymax=100, colors='pink', linestyle='dotted')\n",
    "plt.plot([i + 860 for i in range(len(np_samples[860]['np_test_avg']))], np_samples[860]['np_test_avg'], color='pink', label='neural predictor 860', linewidth=3)\n",
    "\n",
    "ax.vlines(x=344, ymin=0, ymax=100, colors='brown', linestyle='dotted')\n",
    "plt.plot([i + 344 for i in range(len(np_samples[344]['np_test_avg']))], np_samples[344]['np_test_avg'], color='brown', label='neural predictor 344', linewidth=3)\n",
    "\n",
    "ax.vlines(x=172, ymin=0, ymax=100, colors='green', linestyle='dotted')\n",
    "plt.plot([i + 172 for i in range(len(np_samples[172]['np_test_avg']))], np_samples[172]['np_test_avg'], color='green', label='neural predictor 172', linewidth=3)\n",
    "\n",
    "ax.vlines(x=129, ymin=0, ymax=100, colors='orange', linestyle='dotted')\n",
    "plt.plot([i + 129 for i in range(len(np_samples[129]['np_test_avg']))], np_samples[129]['np_test_avg'], color='orange', label='neural predictor 129', linewidth=3)\n",
    "\n",
    "ax.vlines(x=86, ymin=0, ymax=100, colors='purple', linestyle='dotted')\n",
    "plt.plot([i + 86 for i in range(len(np_samples[86]['np_test_avg']))], np_samples[86]['np_test_avg'], color='purple', label='neural predictor 86', linewidth=3)\n",
    "\n",
    "ax.vlines(x=43, ymin=0, ymax=100, colors='yellow', linestyle='dotted')\n",
    "plt.plot([i + 43 for i in range(len(np_samples[43]['np_test_avg']))], np_samples[43]['np_test_avg'], color='yellow', label='neural predictor 43', linewidth=3)\n",
    "\n",
    "\n",
    "# random search plot\n",
    "plt.plot(random_test, color='red', label='random search', linestyle='dashed', linewidth=3)\n",
    "\n",
    "# oracle line\n",
    "ax.hlines(y=oracle_test, xmin=0, xmax=2000, color='black', label='oracle', linewidth=3)\n",
    "\n",
    "# show plot with legend\n",
    "plt.legend(loc='lower right')\n",
    "plt.close()\n",
    "\n",
    "f.savefig('figures/fig7b.pdf', format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "e1037b42-2542-4e7b-91fe-37db1493a696",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(9, 9))\n",
    "\n",
    "# axes settings\n",
    "ax.set(xlim = [0, 500], ylim = [93, 94.75], xlabel='Total training time x 1600 (seconds)', ylabel='Test accuracy %')\n",
    "\n",
    "# neural predictor plot\n",
    "# ax.vlines(x=860, ymin=0, ymax=100, colors='pink', linestyle='dotted')\n",
    "# plt.plot([i + 860 for i in range(len(np_samples[860]['np_test_avg']))], np_samples[860]['np_test_avg'], color='pink', label='neural predictor 860', linewidth=3)\n",
    "\n",
    "ax.vlines(x=344, ymin=0, ymax=100, colors='brown', linestyle='dotted')\n",
    "plt.plot([i + 344 for i in range(len(np_time[344]['np_test_avg']))], np_time[344]['np_test_avg'], color='brown', label='neural predictor 344', linewidth=3)\n",
    "\n",
    "ax.vlines(x=172, ymin=0, ymax=100, colors='green', linestyle='dotted')\n",
    "plt.plot([i + 172 for i in range(len(np_time[172]['np_test_avg']))], np_time[172]['np_test_avg'], color='green', label='neural predictor 172', linewidth=3)\n",
    "\n",
    "ax.vlines(x=129, ymin=0, ymax=100, colors='orange', linestyle='dotted')\n",
    "plt.plot([i + 129 for i in range(len(np_time[129]['np_test_avg']))], np_time[129]['np_test_avg'], color='orange', label='neural predictor 129', linewidth=3)\n",
    "\n",
    "ax.vlines(x=86, ymin=0, ymax=100, colors='purple', linestyle='dotted')\n",
    "plt.plot([i + 86 for i in range(len(np_time[86]['np_test_avg']))], np_time[86]['np_test_avg'], color='purple', label='neural predictor 86', linewidth=3)\n",
    "\n",
    "ax.vlines(x=43, ymin=0, ymax=100, colors='yellow', linestyle='dotted')\n",
    "plt.plot([i + 43 for i in range(len(np_time[43]['np_test_avg']))], np_time[43]['np_test_avg'], color='yellow', label='neural predictor 43', linewidth=3)\n",
    "\n",
    "\n",
    "# random search plot\n",
    "plt.plot(random_test, color='red', label='random search', linestyle='dashed', linewidth=3)\n",
    "\n",
    "# oracle line\n",
    "ax.hlines(y=oracle_test, xmin=0, xmax=2000, color='black', label='oracle', linewidth=3)\n",
    "\n",
    "# show plot with legend\n",
    "plt.legend(loc='lower right')\n",
    "plt.close()\n",
    "\n",
    "f.savefig('figures/fig7c.pdf', format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7607e7ff-042a-485f-9d55-94462234d511",
   "metadata": {},
   "source": [
    "### Ablation Study of Neural Predictor under different architectures (Appendix Fig. 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "6971c05f-f126-40f2-8365-67d0de3b58a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# number of samples vs validation accuracy\n",
    "random_val = np.load('outputs/random_val_avg_2000.npy')\n",
    "random_test = np.load('outputs/random_test_avg_2000.npy')\n",
    "\n",
    "np_cnn = np.load('outputs/ablation_study_by_arch_cnn.npy', allow_pickle=True).ravel()[0]\n",
    "np_mlp = np.load('outputs/ablation_study_by_arch_mlp.npy', allow_pickle=True).ravel()[0]\n",
    "np_gcn = np.load('outputs/ablation_study_by_arch_gcn.npy', allow_pickle=True).ravel()[0]\n",
    "\n",
    "oracle_val = np.load('outputs/oracle_val_acc.npy')\n",
    "oracle_test = np.load('outputs/oracle_test_acc.npy')\n",
    "oracle_val_std = np.load('outputs/oracle_val_std.npy')\n",
    "oracle_test_std = np.load('outputs/oracle_test_std.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "462f43de-be7e-486f-8d78-bef5ecfc8861",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(9, 9))\n",
    "\n",
    "# axes settings\n",
    "ax.set(xlim = [0, 2000], ylim = [92.5, 95.5], xlabel='The number of samples', ylabel='Validation accuracy %',)\n",
    "\n",
    "# neural predictor plot\n",
    "ax.vlines(x=172, ymin=0, ymax=100, colors='green', linestyle='dotted')\n",
    "ax.vlines(x=172, ymin=0, ymax=100, colors='purple', linestyle='dotted')\n",
    "ax.vlines(x=172, ymin=0, ymax=100, colors='blue', linestyle='dotted')\n",
    "\n",
    "plt.plot([i + 172 for i in range(len(np_gcn['np_val_avg']))], np_gcn['np_val_avg'], color='green', label='GCN')\n",
    "plt.fill_between(range(173, 2001), y1=np.array(np_gcn['np_val_avg'])-np.array(np_gcn['np_val_std']), y2=np.array(np_gcn['np_val_avg'])+np.array(np_gcn['np_val_std']), facecolor='green', alpha=0.2)\n",
    "\n",
    "plt.plot([i + 172 for i in range(len(np_mlp['np_val_avg']))], np_mlp['np_val_avg'], color='purple', label='MLP')\n",
    "plt.fill_between(range(173, 2001), y1=np.array(np_mlp['np_val_avg'])-np.array(np_mlp['np_val_std']), y2=np.array(np_mlp['np_val_avg'])+np.array(np_mlp['np_val_std']), facecolor='purple', alpha=0.2)\n",
    "\n",
    "plt.plot([i + 172 for i in range(len(np_cnn['np_val_avg']))], np_cnn['np_val_avg'], color='blue', label='CNN')\n",
    "plt.fill_between(range(173, 2001), y1=np.array(np_cnn['np_val_avg'])-np.array(np_cnn['np_val_std']), y2=np.array(np_cnn['np_val_avg'])+np.array(np_cnn['np_val_std']), facecolor='blue', alpha=0.2)\n",
    "\n",
    "\n",
    "# random search plot\n",
    "plt.plot(random_val, color='red', label='random search', linestyle='dashed')\n",
    "# plt.fill_between(range(1, 501), y1=random_val-random_std, y2=random_val+random_std, facecolor='red', alpha=0.2)\n",
    "\n",
    "# oracle line\n",
    "ax.hlines(y=oracle_val, xmin=0, xmax=2000, color='black', label='oracle')\n",
    "ax.fill_between(range(1, 2001), y1=oracle_val-oracle_val_std, y2=oracle_val+oracle_val_std, facecolor='black', alpha=0.2)\n",
    "\n",
    "# show plot with legend\n",
    "plt.legend(loc='lower right')\n",
    "plt.close()\n",
    "\n",
    "f.savefig('figures/figA1a.pdf', format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "bc75ff88-d844-4af0-b497-3b836aa65f8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(9, 9))\n",
    "\n",
    "# axes settings\n",
    "ax.set(xlim = [0, 2000], ylim = [92, 95], xlabel='The number of samples', ylabel='Test accuracy %',)\n",
    "\n",
    "# neural predictor plot\n",
    "ax.vlines(x=172, ymin=0, ymax=100, colors='green', linestyle='dotted')\n",
    "ax.vlines(x=172, ymin=0, ymax=100, colors='purple', linestyle='dotted')\n",
    "ax.vlines(x=172, ymin=0, ymax=100, colors='blue', linestyle='dotted')\n",
    "\n",
    "plt.plot([i + 172 for i in range(len(np_gcn['np_test_avg']))], np_gcn['np_test_avg'], color='green', label='GCN')\n",
    "plt.fill_between(range(173, 2001), y1=np.array(np_gcn['np_test_avg'])-np.array(np_gcn['np_test_std']), y2=np.array(np_gcn['np_test_avg'])+np.array(np_gcn['np_test_std']), facecolor='green', alpha=0.2)\n",
    "\n",
    "plt.plot([i + 172 for i in range(len(np_mlp['np_test_avg']))], np_mlp['np_test_avg'], color='purple', label='MLP')\n",
    "plt.fill_between(range(173, 2001), y1=np.array(np_mlp['np_test_avg'])-np.array(np_mlp['np_test_std']), y2=np.array(np_mlp['np_test_avg'])+np.array(np_mlp['np_test_std']), facecolor='purple', alpha=0.2)\n",
    "\n",
    "plt.plot([i + 172 for i in range(len(np_cnn['np_test_avg']))], np_cnn['np_test_avg'], color='blue', label='CNN')\n",
    "plt.fill_between(range(173, 2001), y1=np.array(np_cnn['np_test_avg'])-np.array(np_cnn['np_test_std']), y2=np.array(np_cnn['np_test_avg'])+np.array(np_cnn['np_test_std']), facecolor='blue', alpha=0.2)\n",
    "\n",
    "\n",
    "# random search plot\n",
    "plt.plot(random_test, color='red', label='random search', linestyle='dashed')\n",
    "# plt.fill_between(range(1, 501), y1=random_test-random_std, y2=random_test+random_std, facecolor='red', alpha=0.2)\n",
    "\n",
    "# oracle line\n",
    "ax.hlines(y=oracle_test, xmin=0, xmax=2000, color='black', label='oracle')\n",
    "ax.fill_between(range(1, 2001), y1=oracle_test-oracle_test_std, y2=oracle_test+oracle_test_std, facecolor='black', alpha=0.2)\n",
    "\n",
    "# show plot with legend\n",
    "plt.legend(loc='lower right')\n",
    "plt.close()\n",
    "\n",
    "f.savefig('figures/figA1b.pdf', format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "df4c2e6d-ca6c-4ce4-95c8-90f699d8fe1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "np_cnn = np.load('outputs/ablation_study_by_arch_time_cnn.npy', allow_pickle=True).ravel()[0]\n",
    "np_mlp = np.load('outputs/ablation_study_by_arch_time_mlp.npy', allow_pickle=True).ravel()[0]\n",
    "np_gcn = np.load('outputs/ablation_study_by_arch_time_gcn.npy', allow_pickle=True).ravel()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "18ecd818-2eef-4faf-b07e-2dffe421c64d",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(9, 9))\n",
    "\n",
    "# axes settings\n",
    "ax.set(xlim = [0, 500], ylim = [93.2, 94.75], xlabel='Total training time x 1600 (seconds)', ylabel='Test accuracy %')\n",
    "\n",
    "# neural predictor plot\n",
    "ax.vlines(x=500 - len(np_gcn['np_test_avg']), ymin=0, ymax=100, colors='green', linestyle='dotted')\n",
    "ax.vlines(x=500 - len(np_mlp['np_test_avg']), ymin=0, ymax=100, colors='purple', linestyle='dotted')\n",
    "ax.vlines(x=500 - len(np_cnn['np_test_avg']), ymin=0, ymax=100, colors='blue', linestyle='dotted')\n",
    "\n",
    "plt.plot([i + 500 - len(np_gcn['np_test_avg']) for i in range(len(np_gcn['np_test_avg']))], np_gcn['np_test_avg'], color='green', label='GCN')\n",
    "plt.fill_between(range(500 - len(np_gcn['np_test_avg']), 500), y1=np.array(np_gcn['np_test_avg'])-np.array(np_gcn['np_test_std']), y2=np.array(np_gcn['np_test_avg'])+np.array(np_gcn['np_test_std']), facecolor='green', alpha=0.2)\n",
    "\n",
    "plt.plot([i + 500 - len(np_mlp['np_test_avg']) for i in range(len(np_mlp['np_test_avg']))], np_mlp['np_test_avg'], color='purple', label='MLP')\n",
    "plt.fill_between(range(500 - len(np_mlp['np_test_avg']), 500), y1=np.array(np_mlp['np_test_avg'])-np.array(np_mlp['np_test_std']), y2=np.array(np_mlp['np_test_avg'])+np.array(np_mlp['np_test_std']), facecolor='purple', alpha=0.2)\n",
    "\n",
    "plt.plot([i + 500 - len(np_cnn['np_test_avg']) for i in range(len(np_cnn['np_test_avg']))], np_cnn['np_test_avg'], color='blue', label='CNN')\n",
    "plt.fill_between(range(500 - len(np_cnn['np_test_avg']), 500), y1=np.array(np_cnn['np_test_avg'])-np.array(np_cnn['np_test_std']), y2=np.array(np_cnn['np_test_avg'])+np.array(np_cnn['np_test_std']), facecolor='blue', alpha=0.2)\n",
    "\n",
    "\n",
    "# random search plot\n",
    "plt.plot(random_test, color='red', label='random search', linestyle='dashed')\n",
    "# plt.fill_between(range(1, 501), y1=random_test-random_std, y2=random_test+random_std, facecolor='red', alpha=0.2)\n",
    "\n",
    "# oracle line\n",
    "ax.hlines(y=oracle_test, xmin=0, xmax=500, color='black', label='oracle')\n",
    "ax.fill_between(range(1, 501), y1=oracle_test-oracle_test_std, y2=oracle_test+oracle_test_std, facecolor='black', alpha=0.2)\n",
    "\n",
    "# show plot with legend\n",
    "plt.legend(loc='lower right')\n",
    "plt.close()\n",
    "\n",
    "f.savefig('figures/figA1c.pdf', format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "153d3b70-cfa8-41e5-b8f2-501b794a98db",
   "metadata": {},
   "source": [
    "### Performance of Neural Predictor (Fig. 9) -- NAS-Bench-101"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "9e6ae609-5c9a-42a6-ae33-913883bafe96",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set_theme(style=\"whitegrid\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "982516cd-7b4f-491b-a6ab-01e5b0984371",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.load('outputs/nasbench_y_true_train.npy')\n",
    "y_pred = np.load('outputs/nasbench_y_pred_train.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "d545dfe8-0113-41e7-9713-ff53e5e7c22d",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(5, 5))\n",
    "g = sns.scatterplot(x=y_pred.ravel(), y=y_true, color='red', ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Predicted Model Validation Accuracy %', ylabel='Observed Model Validation Accuracy %', xlim=[80, 95.5], ylim=[80, 95.5], title=f'Training samples N={len(y_pred)}')\n",
    "f.savefig('figures/fig9a-nasbench.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "fe02f736-4c27-444d-a137-1954c563b66d",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.load('outputs/nasbench_y_true_val.npy')\n",
    "y_pred = np.load('outputs/nasbench_y_pred_val.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "4d8d4e15-7b29-4b01-9bf4-0603a41baa0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(5, 5))\n",
    "g = sns.scatterplot(x=y_pred.ravel(), y=y_true, ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Predicted Model Validation Accuracy %', ylabel='Observed Model Validation Accuracy %', xlim=[80, 95.5], ylim=[80, 95.5], title=f'Validation samples N={len(y_pred)}')\n",
    "f.savefig('figures/fig9b-nasbench.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "f0348978-22cf-4819-beae-91772b944ddf",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.load('outputs/nasbench_y_true_test_10.npy')\n",
    "y_pred = np.load('outputs/nasbench_y_pred_test_10.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "a60b507e-ec9b-49a8-9ac3-383267fa4c4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(5, 5))\n",
    "g = sns.scatterplot(x=y_pred, y=y_true, color='green', ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Predicted Model Validation Accuracy %', ylabel='Observed Model Validation Accuracy %', xlim=[82.5, 100], ylim=[82.5, 100], title=f'Test samples N={len(y_pred)}')\n",
    "f.savefig('figures/fig9c-nasbench.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c0bb968-2ba9-439d-ad56-9a29fd69dd6a",
   "metadata": {},
   "source": [
    "### Two-stage Neural Predictor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "a292bc95-5b62-47f1-b3c8-1fbb71781893",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.load('outputs/nasbench_two_y_true_train.npy')\n",
    "y_pred = np.load('outputs/nasbench_two_y_pred_train.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "124ae214-2b6a-449e-ba7e-a1dc2832b2ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(5, 5))\n",
    "g = sns.scatterplot(x=y_pred, y=y_true, color='red', ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Predicted Model Validation Accuracy %', ylabel='Observed Model Validation Accuracy %', xlim=[91, 95.5], ylim=[91, 95.5], title=f'Training samples N={len(y_pred)}')\n",
    "f.savefig('figures/fig9a-nashbench-two.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "1c132998-f616-4f55-aaf8-ce79c8087de1",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.load('outputs/nasbench_two_y_true_val.npy')\n",
    "y_pred = np.load('outputs/nasbench_two_y_pred_val.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "48b04bf8-536a-4bf3-82b6-8e7bf8fd5776",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(5, 5))\n",
    "g = sns.scatterplot(x=y_pred, y=y_true, ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Predicted Model Validation Accuracy %', ylabel='Observed Model Validation Accuracy %', xlim=[91, 95.5], ylim=[91, 95.5], title=f'Validation samples N={len(y_pred)}')\n",
    "f.savefig('figures/fig9b-nashbench-two.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "d43dcad4-64a7-4598-b4d6-d7be854fc59b",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.load('outputs/nasbench_two_y_true_test_10.npy')\n",
    "y_pred = np.load('outputs/nasbench_two_y_pred_test_10.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "b57df644-614a-457f-aa9f-5f6120f8b42a",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(5, 5))\n",
    "g = sns.scatterplot(x=y_pred, y=y_true, color='green', ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Predicted Model Validation Accuracy %', ylabel='Observed Model Validation Accuracy %', xlim=[89, 99], ylim=[89, 99], title=f'Test samples N={len(y_pred)}')\n",
    "f.savefig('figures/fig9c-nashbench-two.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9045e76-032b-4a8d-bb6d-45c125bd5ecc",
   "metadata": {},
   "source": [
    "### Performance of Neural Predictor (Fig. 9) -- ProxylessNAS (CIFAR-100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "088d3a04-c687-4005-ab86-28a4a4eed4a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.load('outputs/cifar100_y_true_train.npy')\n",
    "y_pred = np.load('outputs/cifar100_y_pred_train.npy').ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "1b9074dd-4b99-4583-8292-de471f09bbe2",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(5, 5))\n",
    "g = sns.scatterplot(x=y_pred, y=y_true, color='red', ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Predicted Model Validation Accuracy %', ylabel='Observed Model Validation Accuracy %', xlim=[10, 18], ylim=[10, 18], title=f'Training samples N={len(y_pred)}')\n",
    "f.savefig('figures/fig9a-cifar100.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "51fe08d1-d222-482b-8109-ad57b8b3224c",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.load('outputs/cifar100_y_true_val.npy')\n",
    "y_pred = np.load('outputs/cifar100_y_pred_val.npy').ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "9edc6ea7-b476-4b93-ac0f-3d58fe17b2bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(5, 5))\n",
    "g = sns.scatterplot(x=y_pred, y=y_true, ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Predicted Model Validation Accuracy %', ylabel='Observed Model Validation Accuracy %', xlim=[10, 14], ylim=[10, 14], title=f'Validation samples N={len(y_pred)}')\n",
    "f.savefig('figures/fig9b-cifar100.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "2fe1ea7d-cc25-47ea-994d-7b0f5c836ed1",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.load('outputs/cifar100_y_true_test.npy')\n",
    "y_pred = np.load('outputs/cifar100_y_pred_test.npy').ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "b142c620-4b6b-4849-9e90-1cf4a189f37c",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(5, 5))\n",
    "g = sns.scatterplot(x=y_pred, y=y_true, color='green', ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Predicted Model Validation Accuracy %', ylabel='Observed Model Validation Accuracy %', xlim=[10, 14], ylim=[10, 14], title=f'Test samples N={len(y_pred)}')\n",
    "f.savefig('figures/fig9c-cifar100.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2386ec3e-8bc0-4b57-84d8-543d74a582bc",
   "metadata": {},
   "source": [
    "### Performance of Neural Predictor (Fig. 9) -- ProxylessNAS (Caltech-101)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "8e361044-a3f4-4d22-9678-da1723785bae",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.load('outputs/caltech_y_true_train.npy')\n",
    "y_pred = np.load('outputs/caltech_y_pred_train.npy').ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "06bce4e4-8584-4ad7-8404-d1a50853276b",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(5, 5))\n",
    "g = sns.scatterplot(x=y_pred, y=y_true, color='red', ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Predicted Model Validation Accuracy %', ylabel='Observed Model Validation Accuracy %', xlim=[12, 36], ylim=[12, 36], title=f'Training samples N={len(y_pred)}')\n",
    "f.savefig('figures/fig9a-caltech.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "1c2a4e33-88bb-4db2-a319-429c1c4e3ef0",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.load('outputs/caltech_y_true_val.npy')\n",
    "y_pred = np.load('outputs/caltech_y_pred_val.npy').ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "a6703eb1-5204-49de-909d-e5f2384d016a",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(5, 5))\n",
    "g = sns.scatterplot(x=y_pred, y=y_true, ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Predicted Model Validation Accuracy %', ylabel='Observed Model Validation Accuracy %', xlim=[15, 30], ylim=[15, 30], title=f'Validation samples N={len(y_pred)}')\n",
    "f.savefig('figures/fig9b-caltech.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "2891d066-fc4b-4089-bbed-c40302e0d321",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.load('outputs/caltech_y_true_test.npy')\n",
    "y_pred = np.load('outputs/caltech_y_pred_test.npy').ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "24b361c1-3535-43aa-9d5b-75e8e32b76e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(5, 5))\n",
    "g = sns.scatterplot(x=y_pred, y=y_true, color='green', ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Predicted Model Validation Accuracy %', ylabel='Observed Model Validation Accuracy %', xlim=[15, 30], ylim=[15, 30], title=f'Test samples N={len(y_pred)}')\n",
    "f.savefig('figures/fig9c-caltech.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff9dd3a3-c79f-4fad-a3f4-78f26c5eb1cd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "07a561eb-2c16-47c6-8002-31da0385aaa0",
   "metadata": {},
   "source": [
    "### Performance of Neural Predictor (Fig. 9) -- ProxylessNAS (Oxford Pet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "6cff383e-35e5-48ce-bc66-355efceff769",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.load('outputs/oxford_y_true_train.npy')\n",
    "y_pred = np.load('outputs/oxford_y_pred_train.npy').ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "82f1913c-2073-4be5-b952-f7a3e7778d0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(5, 5))\n",
    "g = sns.scatterplot(x=y_pred, y=y_true, color='red', ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Predicted Model Validation Accuracy %', ylabel='Observed Model Validation Accuracy %', xlim=[4, 12], ylim=[4, 12], title=f'Training samples N={len(y_pred)}')\n",
    "f.savefig('figures/fig9a-oxford.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "69f2d3a0-911e-433e-b215-20aaf1692775",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.load('outputs/oxford_y_true_val.npy')\n",
    "y_pred = np.load('outputs/oxford_y_pred_val.npy').ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "c1363154-ff1b-4476-ad16-f7d96dfdbc27",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(5, 5))\n",
    "g = sns.scatterplot(x=y_pred, y=y_true, ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Predicted Model Validation Accuracy %', ylabel='Observed Model Validation Accuracy %', xlim=[6, 12], ylim=[6, 12], title=f'Validation samples N={len(y_pred)}')\n",
    "f.savefig('figures/fig9b-oxford.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "8cf0d1be-bcd1-4ec3-a77d-74b9c7301545",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.load('outputs/oxford_y_true_test.npy')\n",
    "y_pred = np.load('outputs/oxford_y_pred_test.npy').ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "3f4a0460-54f7-4c11-bf9d-2d0676e9dfb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(5, 5))\n",
    "g = sns.scatterplot(x=y_pred, y=y_true, color='green', ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Predicted Model Validation Accuracy %', ylabel='Observed Model Validation Accuracy %', xlim=[6, 10], ylim=[6, 10], title=f'Test samples N={len(y_pred)}')\n",
    "f.savefig('figures/fig9c-oxford.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c17cc16a-2d0c-48ab-bed1-c95c64047ed6",
   "metadata": {},
   "source": [
    "### Performance of Neural Predictor (Fig. 9) -- NAS-Bench-NLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "d228c997-2aca-4e45-a91d-31dfa30645ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.load('outputs/nlp_y_true_train.npy')\n",
    "y_pred = np.load('outputs/nlp_y_pred_train.npy').ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "ad4f0772-b88c-4d69-b8ce-9d14282cd306",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(5, 5))\n",
    "g = sns.scatterplot(x=y_pred, y=y_true, color='red', ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Predicted Model Validation Loss', ylabel='Observed Model Validation Loss', xlim=[4.5, 6], ylim=[4.5, 6], title=f'Training samples N={len(y_pred)}')\n",
    "f.savefig('figures/fig9a-nlp.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "f2a40a34-05e2-4b73-8ded-428e36bff0c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.load('outputs/nlp_y_true_val.npy')\n",
    "y_pred = np.load('outputs/nlp_y_pred_val.npy').ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "b210f6a5-692c-4f4b-9c53-d0f34c7a20d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(5, 5))\n",
    "g = sns.scatterplot(x=y_pred, y=y_true, ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Predicted Model Validation Loss', ylabel='Observed Model Validation Loss', xlim=[4.5, 6], ylim=[4.5, 6], title=f'Validation samples N={len(y_pred)}')\n",
    "f.savefig('figures/fig9b-nlp.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "c39bd1d0-43ab-4be8-bf26-3e74ebd43890",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.load('outputs/nlp_y_true_test_10.npy')\n",
    "y_pred = np.load('outputs/nlp_y_pred_test_10.npy').ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "89eec32f-5f2f-4e65-9f87-927dd7ce70e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(5, 5))\n",
    "g = sns.scatterplot(x=y_pred, y=y_true, color='green', ax=ax)\n",
    "g.plot([0, 1], [0, 1], transform=ax.transAxes, color='lightgray')\n",
    "g.set(xlabel='Predicted Model Validation Loss', ylabel='Observed Model Validation Loss', xlim=[4, 8], ylim=[4, 8], title=f'Test samples N={len(y_pred)}')\n",
    "f.savefig('figures/fig9c-nlp.pdf', format='pdf', bbox_inches='tight')\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f9bfde5-c667-44f1-96ad-61265da64f9b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}