E1-Oracle.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "de324c1a-20ea-4a55-ab3f-c9643b51203a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "from nasbench import api\n",
    "from search_spaces import load_nasbench_101\n",
    "from random_search import run_random_search, random_spec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cd23d261-f6a4-4257-a596-1f909aaa99f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dataset from file... This may take a few minutes...\n",
      "WARNING:tensorflow:From /data2/home/patara/CS470-Project-Team3/nasbench/api.py:146: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use eager execution and: \n",
      "`tf.data.TFRecordDataset(path)`\n",
      "Loaded dataset in 46 seconds\n"
     ]
    }
   ],
   "source": [
    "nasbench = load_nasbench_101()\n",
    "full_metrics = nasbench.computed_statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2416d982-571c-4b51-a1cf-844a786d3580",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "95cbcd5412aa4bbf91e583111723cb16",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/423624 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Each model (point) is the validation accuracy from a single training run. Test accuracies are averaged over three runs. \n",
    "accuracies = []\n",
    "\n",
    "for _, model in tqdm(full_metrics.items()):\n",
    "    metrics = model[108]\n",
    "    metric = random.choice(metrics)\n",
    "    val_acc = metric['final_validation_accuracy']\n",
    "    test_acc = np.average([metric['final_test_accuracy'] for metric in metrics])\n",
    "    \n",
    "    accuracies.append([val_acc, test_acc])\n",
    "accuracies = np.array(accuracies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "97256e03-a46c-474b-b1c1-f0a9ad6c5494",
   "metadata": {},
   "outputs": [],
   "source": [
    "percent_acc = pd.DataFrame(accuracies * 100, columns=['validation', 'test'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "20f72f4e-e426-48d2-adc1-7fdbf789e5f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Validation Accuracy: 95.11218070983887\n",
      "Best Test Accuracy: 94.3175752957662\n"
     ]
    }
   ],
   "source": [
    "# oracle_val, oracle_test = percent_acc['validation'].max(), percent_acc['test'].max()\n",
    "print(f\"Best Validation Accuracy: {percent_acc['validation'].max()}\")\n",
    "print(f\"Best Test Accuracy: {percent_acc['test'].max()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "326a3fa7-2ee2-4ac9-86a7-f971e44c234c",
   "metadata": {},
   "outputs": [],
   "source": [
    "percent_acc.to_csv('outputs/oracle_accuracy.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2bf8eca1-e548-484b-914f-143ff81fe2e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "709929c93a48442a8a56e33aa6a356c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Averaged over 100 oracle experiments, where in each experiment we randomly select one of 3 validation results, \n",
    "# Variance Test for Oracle\n",
    "oracle_val_acc = []\n",
    "oracle_test_acc = []\n",
    "\n",
    "# Repeat oracle method for 100 times\n",
    "for _ in tqdm(range(100)):\n",
    "    val_acc = []\n",
    "    test_acc = []\n",
    "    \n",
    "    for _, model in full_metrics.items():\n",
    "        metrics = model[108]\n",
    "        metric = random.choice(metrics)\n",
    "        val_acc.append(metric['final_validation_accuracy'])\n",
    "        test_acc.append(metric['final_test_accuracy'])\n",
    "    \n",
    "    oracle_val_acc.append(np.max(val_acc))\n",
    "    oracle_test_acc.append(np.max(test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec9cde3e-fec5-4031-b786-8bd72e153857",
   "metadata": {},
   "outputs": [],
   "source": [
    "oracle_val_acc, oracle_val_std = np.average(oracle_val_acc) * 100, np.std(oracle_val_acc) * 100\n",
    "print(\"Oracle Validation Accuracy\")\n",
    "print(f\"avg: {oracle_val_acc} std: {oracle_val_std}\")\n",
    "\n",
    "oracle_test_acc, oracle_test_std = np.average(oracle_test_acc) * 100, np.std(oracle_test_acc) * 100\n",
    "print(\"Oracle Test Accuracy\")\n",
    "print(f\"avg: {oracle_test_acc} std: {oracle_test_std}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e3380d23-7840-4d7d-9188-2bc338d3e9c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('outputs/oracle_val_acc.npy', oracle_val_acc)\n",
    "np.save('outputs/oracle_val_std.npy', oracle_val_std)\n",
    "\n",
    "np.save('outputs/oracle_test_acc.npy', oracle_test_acc)\n",
    "np.save('outputs/oracle_test_std.npy', oracle_test_std)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}