E1-NP-1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8fbbd622-5107-41ac-a4de-1413b60d2665",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "from nasbench import api\n",
    "from search_spaces import load_nasbench_101\n",
    "from random_search import run_random_search, random_spec\n",
    "from neural_predictor import classifier, regressor\n",
    "from input_preprocessing import preprocess_nasbench"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2883042e-302e-4d71-ab34-d0a05da798b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n",
    "os.environ[\"TF_FORCE_GPU_ALLOW_GROWTH\"] = \"true\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9afe4157-edd1-426d-83d2-a2cb5fe03c12",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dataset from file... This may take a few minutes...\n",
      "WARNING:tensorflow:From /home/patara/Courses/CS470-Neural-Predictor/nasbench/api.py:146: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use eager execution and: \n",
      "`tf.data.TFRecordDataset(path)`\n"
     ]
    }
   ],
   "source": [
    "nasbench = load_nasbench_101()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48d3374c-d930-47b4-bdaa-0aad1b10ad0e",
   "metadata": {},
   "source": [
    "### Two-stage vs One-stage prediction 10 split (N=172)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "574d77e8-4f9b-45e7-85bb-d8e9396653e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 172"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d010eb05-a068-4c7f-ba01-2c0d9607b143",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_N_samples(N):\n",
    "    models = []\n",
    "\n",
    "    for _ in range(N):\n",
    "        while True:\n",
    "            model = random_spec(nasbench)\n",
    "            if model not in models:\n",
    "                models.append(nasbench.query(model))\n",
    "                break\n",
    "                \n",
    "    return preprocess_nasbench(models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b4004be-f63d-4337-b784-1ef5f1ea7e4a",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_clf_data = get_N_samples(N)\n",
    "clf = classifier([train_clf_data['X'], train_clf_data['norm_A'], train_clf_data['norm_AT']], train_clf_data['labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7ecde85-cea5-42be-abe1-bc6aab05dac8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cascade_prediction(train_data, test_data):\n",
    "    # training phrase\n",
    "    filtered_train = clf.predict([train_data['X'], train_data['norm_A'], train_data['norm_AT']])\n",
    "    filtered_test = clf.predict([test_data['X'], test_data['norm_A'], test_data['norm_AT']])\n",
    "    \n",
    "    idx = np.where(filtered_train > 0.50)[0]\n",
    "    X = train_data['X'][idx]\n",
    "    A = train_data['norm_A'][idx]\n",
    "    AT = train_data['norm_AT'][idx]\n",
    "    labels = train_data['val_acc'][idx]\n",
    "    \n",
    "    reg = regressor([X, A, AT], labels)\n",
    "    \n",
    "    idx = np.where(filtered_test > 0.50)[0]\n",
    "    X = test_data['X'][idx]\n",
    "    A = test_data['norm_A'][idx]\n",
    "    AT = test_data['norm_AT'][idx]\n",
    "    labels = test_data['val_acc'][idx]\n",
    "    \n",
    "    return reg.evaluate([X, A, AT], labels, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "717fdece-eef6-4e40-8872-690d6cb81dc2",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "two_stage_mse = []\n",
    "one_stage_mse = []\n",
    "\n",
    "for _ in range(10):\n",
    "    train_reg_data = get_N_samples(N)\n",
    "    test_reg_data = get_N_samples(N)\n",
    "    \n",
    "    two_stage_mse.append(cascade_prediction(train_reg_data, test_reg_data))\n",
    "    \n",
    "    reg = regressor([train_reg_data['X'], train_reg_data['norm_A'], train_reg_data['norm_AT']], train_reg_data['val_acc'])\n",
    "    one_stage_mse.append(reg.evaluate([test_reg_data['X'], test_reg_data['norm_A'], test_reg_data['norm_AT']], test_reg_data['val_acc'], verbose=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce85240d-5d53-4936-9681-79b15538de63",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.average(two_stage_mse), np.average(one_stage_mse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4961395e-94f6-410e-b828-3ec400f88b70",
   "metadata": {},
   "outputs": [],
   "source": [
    "LOOPS = 60\n",
    "MAX_SAMPLES = 500\n",
    "MAX_TIME_BUDGET = 8e5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53ba2929-f17e-43b6-822c-f60db149058a",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = get_N_samples(N)\n",
    "reg = regressor([train_data['X'], train_data['norm_A'], train_data['norm_AT']], train_data['val_acc'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "61cb6d1a-7a8b-47a0-83fb-5177a21df7d8",
   "metadata": {},
   "source": [
    "### One-stage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c2d57848-addf-4ee1-bfc3-b15d3dbd265d",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c9c0ffe77e3a420a87f8a22d166cacd0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/328 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "np_val_avg, np_test_avg = [], []\n",
    "np_val_std, np_test_std = [], []\n",
    "\n",
    "val_acc, test_acc = [], []\n",
    "for k in tqdm(range(MAX_SAMPLES - N)):\n",
    "    K = k + 1\n",
    "\n",
    "    loop_val, loop_test = [], []\n",
    "    for _ in range(LOOPS):\n",
    "        test_models = get_N_samples(N+K)\n",
    "        pred_acc = reg.predict([test_models['X'], test_models['norm_A'], test_models['norm_AT']]).ravel()\n",
    "        \n",
    "        topk_idx = tf.math.top_k(pred_acc, k=K).indices.numpy()\n",
    "        selected_val = test_models['val_acc'][topk_idx]\n",
    "        selected_test = test_models['test_acc'][topk_idx]\n",
    "        \n",
    "        best_val_idx = np.argmax(selected_val)\n",
    "        \n",
    "        loop_val.append(selected_val[best_val_idx])\n",
    "        loop_test.append(selected_test[best_val_idx])\n",
    "\n",
    "    val_acc.append(np.mean(loop_val))\n",
    "    test_acc.append(np.mean(loop_test))\n",
    "    \n",
    "    np_val_avg.append(np.max(val_acc))\n",
    "    np_val_std.append(np.std(loop_val))\n",
    "    \n",
    "    np_test_avg.append(np.max(test_acc))\n",
    "    np_test_std.append(np.std(loop_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "66096970-faa8-4e2c-9ec3-8e76f8f2ba4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('outputs/np_val_avg_by_samples.npy', np_val_avg)\n",
    "np.save('outputs/np_val_std_by_samples.npy', np_val_std)\n",
    "np.save('outputs/np_test_avg_by_samples.npy', np_test_avg)\n",
    "np.save('outputs/np_test_std_by_samples.npy', np_test_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "205ad359-b92d-4155-ac08-c24dcbe3f644",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}