bottomupexperiments.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bottom-up Experiments\n",
    "\n",
    "These are the experiments to test the performance of the bottom-up strategy. If you are running an experiment to test the validity of a particular design space, please make sure to load the approrpiate OFA network before doing so. The cells corresponding to each design space have been marked with \"DESIGN SPACE: <NAME>\" at the top of the cell.\n",
    "    \n",
    "We do not recommend running any of the experiments as they may take a while (we average over 10 runs). Please run the cell under the DEMO section to see bottom-up in action. Before running it, make sure you run all cells in the PREP section"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PREP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torchvision import transforms, datasets\n",
    "import numpy as np\n",
    "import time\n",
    "import random\n",
    "import math\n",
    "import copy\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ofa\n",
    "import ofa.model_zoo\n",
    "import ofa.tutorial\n",
    "from ofa.tutorial import LatencyTableReal\n",
    "import ofa.nas.accuracy_predictor.arch_encoder\n",
    "import ofa.nas.search_algorithm\n",
    "import ofa.nas.efficiency_predictor.latency_lookup_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully imported all packages and configured random seed to 1!\n",
      "Using CPU.\n"
     ]
    }
   ],
   "source": [
    "random_seed = 1\n",
    "random.seed(random_seed)\n",
    "np.random.seed(random_seed)\n",
    "torch.manual_seed(random_seed)\n",
    "print('Successfully imported all packages and configured random seed to %d!'%random_seed)\n",
    "\n",
    "cuda_available = torch.cuda.is_available()\n",
    "if cuda_available:\n",
    "    torch.backends.cudnn.enabled = True\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "    torch.cuda.manual_seed(random_seed)\n",
    "    print('Using GPU.')\n",
    "else:\n",
    "    print('Using CPU.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The OFA Network is ready.\n"
     ]
    }
   ],
   "source": [
    "# DESIGN SPACE: MobileNetV3\n",
    "ofa_network = ofa.model_zoo.ofa_net('ofa_mbv3_d234_e346_k357_w1.2', pretrained=True)\n",
    "print('The OFA Network is ready.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://hanlab.mit.edu/files/OnceForAll/tutorial/latency_table@note10/160_lookup_table.yaml\" to /Users/vidhur2k/.hancai/latency_tools/160_lookup_table.yaml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Built latency table for image size: 160.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://hanlab.mit.edu/files/OnceForAll/tutorial/latency_table@note10/176_lookup_table.yaml\" to /Users/vidhur2k/.hancai/latency_tools/176_lookup_table.yaml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Built latency table for image size: 176.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://hanlab.mit.edu/files/OnceForAll/tutorial/latency_table@note10/192_lookup_table.yaml\" to /Users/vidhur2k/.hancai/latency_tools/192_lookup_table.yaml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Built latency table for image size: 192.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://hanlab.mit.edu/files/OnceForAll/tutorial/latency_table@note10/208_lookup_table.yaml\" to /Users/vidhur2k/.hancai/latency_tools/208_lookup_table.yaml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Built latency table for image size: 208.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://hanlab.mit.edu/files/OnceForAll/tutorial/latency_table@note10/224_lookup_table.yaml\" to /Users/vidhur2k/.hancai/latency_tools/224_lookup_table.yaml\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Built latency table for image size: 224.\n",
      "The Latency lookup table on note10 is ready!\n"
     ]
    }
   ],
   "source": [
    "target_hardware = 'note10'\n",
    "latency_table = ofa.tutorial.LatencyTable(device=target_hardware)\n",
    "print('The Latency lookup table on %s is ready!' % target_hardware)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Since GPU is not found in the environment, we skip all scripts related to ImageNet evaluation.\n"
     ]
    }
   ],
   "source": [
    "if cuda_available:\n",
    "    # path to the ImageNet dataset\n",
    "    print(\"Please input the path to the ImageNet dataset.\\n\")\n",
    "    imagenet_data_path = input()\n",
    "    #imagenet_data_path = 'C:\\School\\once-for-all-master\\imgnet'\n",
    "\n",
    "    # if 'imagenet_data_path' is empty, download a subset of ImageNet containing 2000 images (~250M) for test\n",
    "    if not os.path.isdir(imagenet_data_path):\n",
    "        os.makedirs(imagenet_data_path, exist_ok=True)\n",
    "        ofa.utils.download_url('https://hanlab.mit.edu/files/OnceForAll/ofa_cvpr_tutorial/imagenet_1k.zip', model_dir='data')\n",
    "        ! cd data && unzip imagenet_1k 1>/dev/null && cd ..\n",
    "        ! copy -r data/imagenet_1k/* $imagenet_data_path\n",
    "        ! del -rf data\n",
    "        print('%s is empty. Download a subset of ImageNet for test.' % imagenet_data_path)\n",
    "\n",
    "    print('The ImageNet dataset files are ready.')\n",
    "else:\n",
    "    print('Since GPU is not found in the environment, we skip all scripts related to ImageNet evaluation.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Since GPU is not found in the environment, we skip all scripts related to ImageNet evaluation.\n"
     ]
    }
   ],
   "source": [
    "if cuda_available:\n",
    "    # The following function build the data transforms for test\n",
    "    def build_val_transform(size):\n",
    "        return transforms.Compose([\n",
    "            transforms.Resize(int(math.ceil(size / 0.875))),\n",
    "            transforms.CenterCrop(size),\n",
    "            transforms.ToTensor(),\n",
    "            transforms.Normalize(\n",
    "                mean=[0.485, 0.456, 0.406],\n",
    "                std=[0.229, 0.224, 0.225]\n",
    "            ),\n",
    "        ])\n",
    "\n",
    "    data_loader = torch.utils.data.DataLoader(\n",
    "        datasets.ImageFolder(\n",
    "            root=os.path.join(imagenet_data_path, 'val'),\n",
    "            transform=build_val_transform(224)\n",
    "        ),\n",
    "        batch_size=250,  # test batch size\n",
    "        shuffle=True,\n",
    "        num_workers=16,  # number of workers for the data loader\n",
    "        pin_memory=True,\n",
    "        drop_last=False,\n",
    "    )\n",
    "    print('The ImageNet dataloader is ready.')\n",
    "else:\n",
    "    data_loader = None\n",
    "    print('Since GPU is not found in the environment, we skip all scripts related to ImageNet evaluation.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The accuracy predictor is ready!\n"
     ]
    }
   ],
   "source": [
    "#accuracy_predictor = ofa.nas.accuracy_predictor.AccuracyPredictor(\n",
    "accuracy_predictor = ofa.tutorial.AccuracyPredictor(\n",
    "    pretrained=True,\n",
    "    device='cuda:0' if cuda_available else 'cpu'\n",
    ")\n",
    "\n",
    "print('The accuracy predictor is ready!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The OFA Network is ready.\n"
     ]
    }
   ],
   "source": [
    "# DESIGN SPACE: Resnet50D\n",
    "ofa_network = ofa.model_zoo.ofa_net('ofa_resnet50', pretrained=True)\n",
    "print('The OFA Network is ready.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The OFA Network is ready.\n"
     ]
    }
   ],
   "source": [
    "# DESIGN SPACE: ProxylessNAS\n",
    "ofa_network = ofa.model_zoo.ofa_net('ofa_proxyless_d234_e346_k357_w1.3', pretrained=True)\n",
    "print('The OFA Network is ready.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_bottom_up_evolutionary_search(latency_constraint):\n",
    "#     latency_constraint = (20,25,30)  # ms, suggested range [15, 33] ms\n",
    "    P = 100  # The size of population in each generation\n",
    "    N = 500  # How many generations of population to be searched\n",
    "    N2 = 63\n",
    "    r = 0.25  # The ratio of networks that are used as parents for next generation\n",
    "    params = {\n",
    "        'constraint_type': target_hardware, # Let's do FLOPs-constrained search\n",
    "        'efficiency_constraint': latency_constraint,\n",
    "        'mutate_prob': 0.1, # The probability of mutation in evolutionary search\n",
    "        'mutation_ratio': 0.5, # The ratio of networks that are generated through mutation in generation n >= 2.\n",
    "        'efficiency_predictor': latency_table, # To use a predefined efficiency predictor.\n",
    "        'accuracy_predictor': accuracy_predictor, # To use a predefined accuracy_predictor predictor.\n",
    "        'population_size': P,\n",
    "        'max_time_budget': N,\n",
    "        'max_time_budget2': N2,\n",
    "        'parent_ratio': r,\n",
    "    }\n",
    "\n",
    "    # build the evolution finder\n",
    "    finder = ofa.tutorial.EvolutionFinder(**params)\n",
    "\n",
    "    # start searching\n",
    "    result_lis = []\n",
    "    st = time.time()\n",
    "    best_valids, best_info = finder.run_evolution_search_multi()\n",
    "    for i in range(len(latency_constraint)):\n",
    "        result_lis.append(best_info[i])\n",
    "        print('Found best architecture on %s with latency <= %.2f ms'\n",
    "              'It achieves %.2f%s predicted accuracy with %.2f ms latency on %s.' %\n",
    "              (target_hardware, latency_constraint[i], best_info[i][0] * 100, '%', best_info[i][-1], target_hardware))\n",
    "\n",
    "        # visualize the architecture of the searched sub-net\n",
    "        _, net_config, latency = best_info[i]\n",
    "        ofa_network.set_active_subnet(ks=net_config['ks'], d=net_config['d'], e=net_config['e'])\n",
    "        print('Architecture of the searched sub-net:')\n",
    "        print(ofa_network.module_str)\n",
    "    ed = time.time()\n",
    "    print('Time taken for latency constraint: ', (ed-st))\n",
    "    return ed-st"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Running Time for k Latency Constraints"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MobileNetV3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:17<00:00, 28.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.26% predicted accuracy with 14.92 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E4.0, K3), Identity)\n",
      "(SE(O48, E3.0, K5), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "((O96, E4.0, K7), None)\n",
      "((O96, E3.0, K7), Identity)\n",
      "((O96, E4.0, K3), Identity)\n",
      "(SE(O136, E4.0, K3), None)\n",
      "(SE(O136, E4.0, K7), Identity)\n",
      "(SE(O136, E3.0, K5), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  26.16634511947632\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:19<00:00, 25.39it/s]\n",
      "Searching with note10 constraint (20): 100%|██████████| 100/100 [00:03<00:00, 26.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.64% predicted accuracy with 14.90 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E4.0, K3), Identity)\n",
      "((O96, E4.0, K5), Identity)\n",
      "(SE(O136, E4.0, K3), None)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O192, E4.0, K3), None)\n",
      "(SE(O192, E4.0, K3), Identity)\n",
      "(SE(O192, E3.0, K3), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.30% predicted accuracy with 19.72 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E4.0, K3), Identity)\n",
      "((O96, E4.0, K5), Identity)\n",
      "(SE(O136, E4.0, K3), None)\n",
      "(SE(O136, E6.0, K5), Identity)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O192, E4.0, K3), None)\n",
      "(SE(O192, E4.0, K3), Identity)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  31.591012001037598\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:21<00:00, 23.06it/s]\n",
      "Searching with note10 constraint (20): 100%|██████████| 100/100 [00:04<00:00, 23.96it/s]\n",
      "Searching with note10 constraint (25): 100%|██████████| 100/100 [00:03<00:00, 26.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.52% predicted accuracy with 14.99 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K3), None)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "(SE(O48, E3.0, K5), Identity)\n",
      "((O96, E6.0, K5), None)\n",
      "((O96, E4.0, K3), Identity)\n",
      "((O96, E3.0, K5), Identity)\n",
      "(SE(O136, E4.0, K5), None)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O136, E3.0, K5), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.33% predicted accuracy with 19.96 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K3), None)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "(SE(O48, E3.0, K5), Identity)\n",
      "((O96, E6.0, K5), None)\n",
      "((O96, E4.0, K5), Identity)\n",
      "((O96, E3.0, K5), Identity)\n",
      "((O96, E3.0, K5), Identity)\n",
      "(SE(O136, E6.0, K7), None)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O136, E3.0, K5), Identity)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K7), Identity)\n",
      "(SE(O192, E3.0, K5), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 25.00 msIt achieves 81.57% predicted accuracy with 24.96 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K3), None)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "((O96, E6.0, K5), None)\n",
      "((O96, E4.0, K5), Identity)\n",
      "((O96, E6.0, K5), Identity)\n",
      "((O96, E3.0, K7), Identity)\n",
      "(SE(O136, E6.0, K7), None)\n",
      "(SE(O136, E4.0, K7), Identity)\n",
      "(SE(O136, E3.0, K5), Identity)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K7), Identity)\n",
      "(SE(O192, E4.0, K5), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  38.2353081703186\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:21<00:00, 23.31it/s]\n",
      "Searching with note10 constraint (20): 100%|██████████| 100/100 [00:04<00:00, 24.74it/s]\n",
      "Searching with note10 constraint (25): 100%|██████████| 100/100 [00:04<00:00, 24.66it/s]\n",
      "Searching with note10 constraint (30): 100%|██████████| 100/100 [00:04<00:00, 24.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.49% predicted accuracy with 14.90 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K3), None)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E3.0, K3), Identity)\n",
      "(SE(O136, E4.0, K3), None)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O192, E4.0, K3), None)\n",
      "(SE(O192, E4.0, K3), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.31% predicted accuracy with 19.97 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K3), None)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E4.0, K3), Identity)\n",
      "(SE(O136, E4.0, K3), None)\n",
      "(SE(O136, E3.0, K7), Identity)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O192, E4.0, K3), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E4.0, K3), Identity)\n",
      "(SE(O192, E4.0, K7), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 25.00 msIt achieves 81.71% predicted accuracy with 24.75 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E3.0, K5), Identity)\n",
      "((O96, E3.0, K3), Identity)\n",
      "((O96, E4.0, K3), Identity)\n",
      "(SE(O136, E6.0, K3), None)\n",
      "(SE(O136, E3.0, K5), Identity)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E4.0, K7), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 30.00 msIt achieves 82.38% predicted accuracy with 29.97 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E6.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E4.0, K7), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E3.0, K5), Identity)\n",
      "((O96, E3.0, K3), Identity)\n",
      "((O96, E4.0, K3), Identity)\n",
      "(SE(O136, E6.0, K3), None)\n",
      "(SE(O136, E3.0, K5), Identity)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O136, E3.0, K7), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E4.0, K7), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  43.80901288986206\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:22<00:00, 22.33it/s]\n",
      "Searching with note10 constraint (20): 100%|██████████| 100/100 [00:04<00:00, 23.28it/s]\n",
      "Searching with note10 constraint (25): 100%|██████████| 100/100 [00:03<00:00, 25.44it/s]\n",
      "Searching with note10 constraint (30): 100%|██████████| 100/100 [00:03<00:00, 26.36it/s]\n",
      "Searching with note10 constraint (35): 100%|██████████| 100/100 [00:04<00:00, 24.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.67% predicted accuracy with 14.88 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E4.0, K3), Identity)\n",
      "((O96, E4.0, K5), Identity)\n",
      "(SE(O136, E4.0, K3), None)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O192, E4.0, K3), None)\n",
      "(SE(O192, E4.0, K3), Identity)\n",
      "(SE(O192, E3.0, K3), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.29% predicted accuracy with 19.85 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E4.0, K3), Identity)\n",
      "((O96, E4.0, K5), Identity)\n",
      "(SE(O136, E4.0, K3), None)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O192, E4.0, K3), None)\n",
      "(SE(O192, E4.0, K3), Identity)\n",
      "(SE(O192, E3.0, K5), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 25.00 msIt achieves 81.85% predicted accuracy with 24.89 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E3.0, K7), Identity)\n",
      "((O96, E4.0, K3), Identity)\n",
      "(SE(O136, E6.0, K3), None)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 30.00 msIt achieves 82.82% predicted accuracy with 29.57 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E3.0, K7), Identity)\n",
      "((O96, E4.0, K3), Identity)\n",
      "((O96, E3.0, K5), Identity)\n",
      "(SE(O136, E6.0, K7), None)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O136, E6.0, K3), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E4.0, K5), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 35.00 msIt achieves 83.34% predicted accuracy with 34.92 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E3.0, K7), Identity)\n",
      "((O32, E4.0, K3), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "(SE(O48, E3.0, K5), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E3.0, K7), Identity)\n",
      "((O96, E4.0, K5), Identity)\n",
      "((O96, E3.0, K7), Identity)\n",
      "(SE(O136, E6.0, K7), None)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O136, E6.0, K3), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E3.0, K7), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  47.78053402900696\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "latency_constraints = (15, 20, 25, 30, 35)\n",
    "times = {}\n",
    "for i in range(len(latency_constraints)):\n",
    "    latency_constraint = latency_constraints[:i+1]\n",
    "    temp_times = 0\n",
    "    for j in range(10):\n",
    "        temp_times += run_bottom_up_evolutionary_search(latency_constraint)\n",
    "    times[latency_constraint] /= 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:21<00:00, 23.20it/s]\n",
      "Searching with note10 constraint (20): 100%|██████████| 100/100 [00:03<00:00, 25.27it/s]\n",
      "Searching with note10 constraint (25): 100%|██████████| 100/100 [00:03<00:00, 25.85it/s]\n",
      "Searching with note10 constraint (30): 100%|██████████| 100/100 [00:03<00:00, 25.12it/s]\n",
      "Searching with note10 constraint (35): 100%|██████████| 100/100 [00:03<00:00, 25.02it/s]\n",
      "Searching with note10 constraint (40): 100%|██████████| 100/100 [00:03<00:00, 27.40it/s]\n",
      "Searching with note10 constraint (45): 100%|██████████| 100/100 [00:03<00:00, 30.29it/s]\n",
      "Searching with note10 constraint (50): 100%|██████████| 100/100 [00:03<00:00, 27.78it/s]\n",
      "Searching with note10 constraint (55): 100%|██████████| 100/100 [00:03<00:00, 31.75it/s]\n",
      "Searching with note10 constraint (60): 100%|██████████| 100/100 [00:03<00:00, 28.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.51% predicted accuracy with 14.92 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K3), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "((O96, E4.0, K7), None)\n",
      "((O96, E4.0, K5), Identity)\n",
      "(SE(O136, E4.0, K7), None)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E3.0, K3), Identity)\n",
      "(SE(O192, E4.0, K3), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.56% predicted accuracy with 20.00 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E6.0, K3), None)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "((O96, E4.0, K7), None)\n",
      "((O96, E4.0, K5), Identity)\n",
      "(SE(O136, E4.0, K7), None)\n",
      "(SE(O136, E4.0, K7), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E4.0, K3), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 25.00 msIt achieves 81.85% predicted accuracy with 24.84 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E3.0, K5), None)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "((O96, E4.0, K7), None)\n",
      "((O96, E4.0, K7), Identity)\n",
      "((O96, E6.0, K3), Identity)\n",
      "(SE(O136, E4.0, K5), None)\n",
      "(SE(O136, E4.0, K7), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E3.0, K3), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 30.00 msIt achieves 82.50% predicted accuracy with 29.96 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E3.0, K5), None)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "(SE(O48, E3.0, K7), Identity)\n",
      "((O96, E4.0, K7), None)\n",
      "((O96, E3.0, K7), Identity)\n",
      "((O96, E6.0, K3), Identity)\n",
      "(SE(O136, E4.0, K5), None)\n",
      "(SE(O136, E4.0, K7), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E3.0, K5), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 35.00 msIt achieves 83.28% predicted accuracy with 34.87 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E6.0, K3), None)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "(SE(O48, E3.0, K7), Identity)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E3.0, K7), Identity)\n",
      "((O96, E4.0, K3), Identity)\n",
      "(SE(O136, E4.0, K5), None)\n",
      "(SE(O136, E4.0, K7), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O136, E6.0, K5), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E3.0, K5), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 40.00 msIt achieves 83.78% predicted accuracy with 39.60 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E6.0, K3), None)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "(SE(O48, E3.0, K7), Identity)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E3.0, K7), Identity)\n",
      "((O96, E4.0, K3), Identity)\n",
      "(SE(O136, E4.0, K7), None)\n",
      "(SE(O136, E4.0, K7), Identity)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O136, E6.0, K5), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E3.0, K5), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 45.00 msIt achieves 84.10% predicted accuracy with 44.60 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E6.0, K5), None)\n",
      "(SE(O48, E6.0, K7), Identity)\n",
      "(SE(O48, E3.0, K7), Identity)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E3.0, K7), Identity)\n",
      "((O96, E4.0, K3), Identity)\n",
      "(SE(O136, E4.0, K7), None)\n",
      "(SE(O136, E4.0, K7), Identity)\n",
      "(SE(O136, E6.0, K7), Identity)\n",
      "(SE(O136, E6.0, K5), Identity)\n",
      "(SE(O192, E6.0, K7), None)\n",
      "(SE(O192, E6.0, K7), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E3.0, K5), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 50.00 msIt achieves 84.47% predicted accuracy with 49.50 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E6.0, K3), Identity)\n",
      "(SE(O48, E6.0, K5), None)\n",
      "(SE(O48, E6.0, K7), Identity)\n",
      "(SE(O48, E3.0, K7), Identity)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E3.0, K7), Identity)\n",
      "((O96, E4.0, K3), Identity)\n",
      "(SE(O136, E4.0, K7), None)\n",
      "(SE(O136, E4.0, K7), Identity)\n",
      "(SE(O136, E6.0, K7), Identity)\n",
      "(SE(O136, E6.0, K5), Identity)\n",
      "(SE(O192, E6.0, K7), None)\n",
      "(SE(O192, E6.0, K7), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E3.0, K5), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 55.00 msIt achieves 84.62% predicted accuracy with 52.33 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E6.0, K3), Identity)\n",
      "(SE(O48, E6.0, K5), None)\n",
      "(SE(O48, E6.0, K5), Identity)\n",
      "(SE(O48, E3.0, K7), Identity)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E3.0, K7), Identity)\n",
      "((O96, E6.0, K3), Identity)\n",
      "((O96, E4.0, K7), Identity)\n",
      "(SE(O136, E4.0, K7), None)\n",
      "(SE(O136, E6.0, K7), Identity)\n",
      "(SE(O136, E6.0, K7), Identity)\n",
      "(SE(O136, E6.0, K5), Identity)\n",
      "(SE(O192, E6.0, K7), None)\n",
      "(SE(O192, E6.0, K7), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E3.0, K5), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 60.00 msIt achieves 84.67% predicted accuracy with 59.12 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K7), Identity)\n",
      "((O32, E6.0, K3), Identity)\n",
      "((O32, E4.0, K3), Identity)\n",
      "(SE(O48, E6.0, K5), None)\n",
      "(SE(O48, E6.0, K7), Identity)\n",
      "(SE(O48, E3.0, K7), Identity)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E3.0, K7), Identity)\n",
      "((O96, E6.0, K3), Identity)\n",
      "((O96, E6.0, K7), Identity)\n",
      "(SE(O136, E4.0, K5), None)\n",
      "(SE(O136, E6.0, K7), Identity)\n",
      "(SE(O136, E6.0, K7), Identity)\n",
      "(SE(O136, E6.0, K5), Identity)\n",
      "(SE(O192, E6.0, K7), None)\n",
      "(SE(O192, E6.0, K7), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E3.0, K5), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  64.29084706306458\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "latency_constraints = (15, 20, 25, 30, 35, 40, 45, 50, 55, 60)\n",
    "temp_times = 0\n",
    "for j in range(10):\n",
    "    temp_times += run_bottom_up_evolutionary_search(latency_constraints)\n",
    "times[latency_constraints] /= 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(15,): 26.16634511947632,\n",
       " (15, 20): 31.591012001037598,\n",
       " (15, 20, 25): 38.2353081703186,\n",
       " (15, 20, 25, 30): 43.80901288986206,\n",
       " (15, 20, 25, 30, 35): 47.78053402900696,\n",
       " (15, 20, 25, 30, 35, 40, 45, 50, 55, 60): 64.29084706306458}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "times"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ResNet50D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:21<00:00, 23.12it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.63% predicted accuracy with 14.88 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1024->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->6144->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Time taken for latency constraint:  30.703929901123047\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:23<00:00, 21.26it/s]\n",
      "Searching with note10 constraint (20): 100%|██████████| 100/100 [00:04<00:00, 21.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.68% predicted accuracy with 14.96 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->6144->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.58% predicted accuracy with 19.98 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1024->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Time taken for latency constraint:  37.73849034309387\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:21<00:00, 23.45it/s]\n",
      "Searching with note10 constraint (20): 100%|██████████| 100/100 [00:03<00:00, 25.70it/s]\n",
      "Searching with note10 constraint (25): 100%|██████████| 100/100 [00:03<00:00, 25.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.54% predicted accuracy with 14.97 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->6144->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.35% predicted accuracy with 19.98 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->1024->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->6144->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 25.00 msIt achieves 81.60% predicted accuracy with 24.99 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->1024->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Time taken for latency constraint:  38.1832230091095\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:23<00:00, 21.39it/s]\n",
      "Searching with note10 constraint (20): 100%|██████████| 100/100 [00:04<00:00, 23.95it/s]\n",
      "Searching with note10 constraint (25): 100%|██████████| 100/100 [00:03<00:00, 26.70it/s]\n",
      "Searching with note10 constraint (30): 100%|██████████| 100/100 [00:03<00:00, 27.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.48% predicted accuracy with 14.94 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1024->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->6144->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.24% predicted accuracy with 19.91 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->6144->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 25.00 msIt achieves 81.47% predicted accuracy with 24.95 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 30.00 msIt achieves 82.23% predicted accuracy with 29.99 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Time taken for latency constraint:  43.44956994056702\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:22<00:00, 22.47it/s]\n",
      "Searching with note10 constraint (20): 100%|██████████| 100/100 [00:03<00:00, 26.03it/s]\n",
      "Searching with note10 constraint (25): 100%|██████████| 100/100 [00:03<00:00, 25.96it/s]\n",
      "Searching with note10 constraint (30): 100%|██████████| 100/100 [00:03<00:00, 26.07it/s]\n",
      "Searching with note10 constraint (35): 100%|██████████| 100/100 [00:03<00:00, 27.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.48% predicted accuracy with 14.94 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1024->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1024->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->6144->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.61% predicted accuracy with 19.74 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1024->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 25.00 msIt achieves 81.84% predicted accuracy with 24.78 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 30.00 msIt achieves 82.35% predicted accuracy with 29.92 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 35.00 msIt achieves 83.23% predicted accuracy with 34.94 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->1024->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Time taken for latency constraint:  47.836430072784424\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "latency_constraints = (15, 20, 25, 30, 35)\n",
    "times = {}\n",
    "for i in range(len(latency_constraints)):\n",
    "    latency_constraint = latency_constraints[:i+1]\n",
    "    times[latency_constraint] = run_bottom_up_evolutionary_search(latency_constraint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:19<00:00, 25.10it/s]\n",
      "Searching with note10 constraint (20): 100%|██████████| 100/100 [00:03<00:00, 25.17it/s]\n",
      "Searching with note10 constraint (25): 100%|██████████| 100/100 [00:03<00:00, 25.37it/s]\n",
      "Searching with note10 constraint (30): 100%|██████████| 100/100 [00:03<00:00, 26.43it/s]\n",
      "Searching with note10 constraint (35): 100%|██████████| 100/100 [00:03<00:00, 27.82it/s]\n",
      "Searching with note10 constraint (40): 100%|██████████| 100/100 [00:03<00:00, 27.52it/s]\n",
      "Searching with note10 constraint (45): 100%|██████████| 100/100 [00:04<00:00, 23.97it/s]\n",
      "Searching with note10 constraint (50): 100%|██████████| 100/100 [00:03<00:00, 27.74it/s]\n",
      "Searching with note10 constraint (55): 100%|██████████| 100/100 [00:03<00:00, 29.63it/s]\n",
      "Searching with note10 constraint (60): 100%|██████████| 100/100 [00:03<00:00, 30.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.43% predicted accuracy with 14.93 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->6144->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.65% predicted accuracy with 20.00 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->6144->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 25.00 msIt achieves 82.05% predicted accuracy with 24.98 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 30.00 msIt achieves 82.55% predicted accuracy with 29.71 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 35.00 msIt achieves 83.18% predicted accuracy with 34.72 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->1024->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 40.00 msIt achieves 84.06% predicted accuracy with 39.49 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->1024->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 45.00 msIt achieves 84.36% predicted accuracy with 44.96 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->1024->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 50.00 msIt achieves 84.50% predicted accuracy with 49.36 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->1024->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 55.00 msIt achieves 84.53% predicted accuracy with 50.43 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->1024->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 60.00 msIt achieves 84.55% predicted accuracy with 53.10 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->1024->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Time taken for latency constraint:  63.43661594390869\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "latency_constraints = (15, 20, 25, 30, 35, 40, 45, 50, 55, 60)\n",
    "times[latency_constraints] = run_bottom_up_evolutionary_search(latency_constraints)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(15,): 30.703929901123047,\n",
       " (15, 20): 37.73849034309387,\n",
       " (15, 20, 25): 38.1832230091095,\n",
       " (15, 20, 25, 30): 43.44956994056702,\n",
       " (15, 20, 25, 30, 35): 47.836430072784424,\n",
       " (15, 20, 25, 30, 35, 40, 45, 50, 55, 60): 63.43661594390869}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "times"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ProxylessNAS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:20<00:00, 23.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.73% predicted accuracy with 14.99 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K3), None)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O104, E4.0, K5), None)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O104, E3.0, K3), Identity)\n",
      "((O128, E4.0, K3), None)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O128, E3.0, K5), Identity)\n",
      "((O248, E4.0, K3), None)\n",
      "((O248, E6.0, K3), Identity)\n",
      "((O248, E4.0, K7), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  29.050503969192505\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:22<00:00, 22.26it/s]\n",
      "Searching with note10 constraint (20): 100%|██████████| 100/100 [00:04<00:00, 22.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.48% predicted accuracy with 14.94 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E4.0, K3), Identity)\n",
      "((O56, E3.0, K3), None)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O104, E6.0, K3), None)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O104, E4.0, K5), Identity)\n",
      "((O128, E4.0, K5), None)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O128, E3.0, K3), Identity)\n",
      "((O248, E4.0, K3), None)\n",
      "((O248, E4.0, K3), Identity)\n",
      "((O248, E4.0, K7), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.41% predicted accuracy with 19.97 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K3), None)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O56, E3.0, K5), Identity)\n",
      "((O104, E6.0, K5), None)\n",
      "((O104, E3.0, K7), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O128, E6.0, K5), None)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E4.0, K3), Identity)\n",
      "((O248, E4.0, K5), Identity)\n",
      "((O248, E3.0, K7), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  35.075645208358765\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:21<00:00, 22.95it/s]\n",
      "Searching with note10 constraint (20): 100%|██████████| 100/100 [00:04<00:00, 23.99it/s]\n",
      "Searching with note10 constraint (25): 100%|██████████| 100/100 [00:04<00:00, 24.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.55% predicted accuracy with 14.99 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K5), None)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O56, E3.0, K5), Identity)\n",
      "((O104, E6.0, K5), None)\n",
      "((O104, E4.0, K7), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O128, E4.0, K3), None)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O128, E3.0, K5), Identity)\n",
      "((O248, E4.0, K3), None)\n",
      "((O248, E6.0, K3), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.25% predicted accuracy with 19.96 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K3), None)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O56, E3.0, K5), Identity)\n",
      "((O104, E6.0, K5), None)\n",
      "((O104, E4.0, K5), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O128, E4.0, K7), None)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O128, E3.0, K5), Identity)\n",
      "((O128, E4.0, K7), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E6.0, K3), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 25.00 msIt achieves 81.73% predicted accuracy with 24.90 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K3), None)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O104, E6.0, K5), None)\n",
      "((O104, E4.0, K5), Identity)\n",
      "((O104, E3.0, K3), Identity)\n",
      "((O128, E4.0, K7), None)\n",
      "((O128, E4.0, K7), Identity)\n",
      "((O128, E3.0, K5), Identity)\n",
      "((O128, E4.0, K5), Identity)\n",
      "((O248, E6.0, K5), None)\n",
      "((O248, E6.0, K3), Identity)\n",
      "((O248, E6.0, K7), Identity)\n",
      "((O248, E4.0, K5), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  40.246270179748535\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:26<00:00, 18.77it/s]\n",
      "Searching with note10 constraint (20): 100%|██████████| 100/100 [00:05<00:00, 18.86it/s]\n",
      "Searching with note10 constraint (25): 100%|██████████| 100/100 [00:05<00:00, 17.22it/s]\n",
      "Searching with note10 constraint (30): 100%|██████████| 100/100 [00:05<00:00, 18.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.63% predicted accuracy with 14.96 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K5), None)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O104, E6.0, K5), None)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O104, E3.0, K7), Identity)\n",
      "((O104, E3.0, K3), Identity)\n",
      "((O128, E3.0, K3), None)\n",
      "((O128, E3.0, K5), Identity)\n",
      "((O128, E4.0, K5), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E3.0, K7), Identity)\n",
      "((O248, E3.0, K3), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.51% predicted accuracy with 19.74 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K5), None)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O104, E4.0, K5), None)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O104, E3.0, K3), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O128, E3.0, K3), None)\n",
      "((O128, E6.0, K5), Identity)\n",
      "((O128, E4.0, K5), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E3.0, K5), Identity)\n",
      "((O248, E4.0, K3), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 25.00 msIt achieves 81.69% predicted accuracy with 24.89 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K5), None)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O104, E4.0, K5), None)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O104, E3.0, K3), Identity)\n",
      "((O104, E6.0, K5), Identity)\n",
      "((O128, E3.0, K3), None)\n",
      "((O128, E6.0, K5), Identity)\n",
      "((O128, E4.0, K5), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O248, E6.0, K5), None)\n",
      "((O248, E3.0, K5), Identity)\n",
      "((O248, E4.0, K3), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 30.00 msIt achieves 82.26% predicted accuracy with 29.79 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K5), None)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O104, E4.0, K5), None)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O104, E3.0, K3), Identity)\n",
      "((O104, E6.0, K5), Identity)\n",
      "((O128, E3.0, K3), None)\n",
      "((O128, E6.0, K5), Identity)\n",
      "((O128, E4.0, K5), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O248, E6.0, K5), None)\n",
      "((O248, E3.0, K5), Identity)\n",
      "((O248, E4.0, K3), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  52.753483057022095\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:25<00:00, 19.27it/s]\n",
      "Searching with note10 constraint (20): 100%|██████████| 100/100 [00:05<00:00, 18.18it/s]\n",
      "Searching with note10 constraint (25): 100%|██████████| 100/100 [00:04<00:00, 22.39it/s]\n",
      "Searching with note10 constraint (30): 100%|██████████| 100/100 [00:04<00:00, 23.19it/s]\n",
      "Searching with note10 constraint (35): 100%|██████████| 100/100 [00:03<00:00, 25.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.75% predicted accuracy with 14.94 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K3), None)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O56, E3.0, K5), Identity)\n",
      "((O104, E3.0, K5), None)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O128, E4.0, K3), None)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O128, E3.0, K5), Identity)\n",
      "((O248, E4.0, K3), None)\n",
      "((O248, E4.0, K3), Identity)\n",
      "((O248, E4.0, K7), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.30% predicted accuracy with 20.00 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K5), None)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O56, E3.0, K5), Identity)\n",
      "((O104, E6.0, K5), None)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O128, E4.0, K3), None)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O128, E6.0, K5), Identity)\n",
      "((O248, E4.0, K7), None)\n",
      "((O248, E4.0, K7), Identity)\n",
      "((O248, E4.0, K3), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 25.00 msIt achieves 81.73% predicted accuracy with 24.96 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K3), None)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O56, E3.0, K5), Identity)\n",
      "((O104, E6.0, K5), None)\n",
      "((O104, E4.0, K7), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O128, E6.0, K3), None)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O128, E6.0, K5), Identity)\n",
      "((O248, E4.0, K7), None)\n",
      "((O248, E6.0, K3), Identity)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 30.00 msIt achieves 82.18% predicted accuracy with 29.96 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K3), None)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O56, E3.0, K5), Identity)\n",
      "((O104, E6.0, K5), None)\n",
      "((O104, E4.0, K7), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O128, E6.0, K3), None)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O128, E6.0, K5), Identity)\n",
      "((O248, E4.0, K7), None)\n",
      "((O248, E6.0, K3), Identity)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 35.00 msIt achieves 82.95% predicted accuracy with 34.36 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K3), None)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O56, E3.0, K5), Identity)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O104, E6.0, K5), None)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O128, E6.0, K3), None)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O128, E6.0, K7), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O248, E4.0, K7), None)\n",
      "((O248, E6.0, K7), Identity)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O248, E3.0, K7), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  56.32675790786743\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "latency_constraints = (15, 20, 25, 30, 35)\n",
    "times = {}\n",
    "for i in range(len(latency_constraints)):\n",
    "    latency_constraint = latency_constraints[:i+1]\n",
    "    times[latency_constraint] = run_bottom_up_evolutionary_search(latency_constraint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:25<00:00, 20.00it/s]\n",
      "Searching with note10 constraint (20): 100%|██████████| 100/100 [00:04<00:00, 20.16it/s]\n",
      "Searching with note10 constraint (25): 100%|██████████| 100/100 [00:04<00:00, 20.26it/s]\n",
      "Searching with note10 constraint (30): 100%|██████████| 100/100 [00:04<00:00, 21.14it/s]\n",
      "Searching with note10 constraint (35): 100%|██████████| 100/100 [00:04<00:00, 23.17it/s]\n",
      "Searching with note10 constraint (40): 100%|██████████| 100/100 [00:04<00:00, 20.42it/s]\n",
      "Searching with note10 constraint (45): 100%|██████████| 100/100 [00:05<00:00, 19.51it/s]\n",
      "Searching with note10 constraint (50): 100%|██████████| 100/100 [00:04<00:00, 24.36it/s]\n",
      "Searching with note10 constraint (55): 100%|██████████| 100/100 [00:04<00:00, 23.85it/s]\n",
      "Searching with note10 constraint (60): 100%|██████████| 100/100 [00:03<00:00, 25.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.51% predicted accuracy with 14.97 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K5), None)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O104, E4.0, K7), None)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O104, E3.0, K7), Identity)\n",
      "((O128, E3.0, K3), None)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E3.0, K3), Identity)\n",
      "((O248, E3.0, K3), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.43% predicted accuracy with 19.99 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K5), None)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O104, E4.0, K5), None)\n",
      "((O104, E4.0, K5), Identity)\n",
      "((O104, E3.0, K3), Identity)\n",
      "((O128, E6.0, K7), None)\n",
      "((O128, E4.0, K5), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E3.0, K7), Identity)\n",
      "((O248, E6.0, K3), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 25.00 msIt achieves 81.95% predicted accuracy with 24.97 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K5), None)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O104, E4.0, K5), None)\n",
      "((O104, E4.0, K5), Identity)\n",
      "((O104, E3.0, K3), Identity)\n",
      "((O128, E4.0, K7), None)\n",
      "((O128, E4.0, K5), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 30.00 msIt achieves 82.89% predicted accuracy with 29.82 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K5), None)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O56, E6.0, K3), Identity)\n",
      "((O104, E4.0, K5), None)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O104, E3.0, K3), Identity)\n",
      "((O104, E3.0, K3), Identity)\n",
      "((O128, E6.0, K7), None)\n",
      "((O128, E4.0, K7), Identity)\n",
      "((O128, E6.0, K3), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E6.0, K3), Identity)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O248, E4.0, K7), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 35.00 msIt achieves 83.29% predicted accuracy with 33.04 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E4.0, K3), Identity)\n",
      "((O56, E4.0, K5), None)\n",
      "((O56, E6.0, K3), Identity)\n",
      "((O56, E6.0, K3), Identity)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O104, E4.0, K5), None)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O104, E3.0, K3), Identity)\n",
      "((O104, E3.0, K3), Identity)\n",
      "((O128, E6.0, K7), None)\n",
      "((O128, E4.0, K7), Identity)\n",
      "((O128, E6.0, K3), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O248, E6.0, K5), None)\n",
      "((O248, E6.0, K3), Identity)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O248, E4.0, K7), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 40.00 msIt achieves 83.84% predicted accuracy with 39.85 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E4.0, K3), Identity)\n",
      "((O56, E4.0, K5), None)\n",
      "((O56, E6.0, K3), Identity)\n",
      "((O56, E6.0, K3), Identity)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O104, E4.0, K5), None)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O104, E3.0, K3), Identity)\n",
      "((O104, E3.0, K3), Identity)\n",
      "((O128, E6.0, K7), None)\n",
      "((O128, E4.0, K7), Identity)\n",
      "((O128, E6.0, K3), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O248, E6.0, K7), None)\n",
      "((O248, E6.0, K3), Identity)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O248, E4.0, K7), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 45.00 msIt achieves 84.15% predicted accuracy with 44.93 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E4.0, K3), Identity)\n",
      "((O56, E6.0, K5), None)\n",
      "((O56, E6.0, K3), Identity)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O56, E3.0, K5), Identity)\n",
      "((O104, E4.0, K5), None)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O104, E3.0, K3), Identity)\n",
      "((O104, E3.0, K7), Identity)\n",
      "((O128, E6.0, K7), None)\n",
      "((O128, E4.0, K7), Identity)\n",
      "((O128, E6.0, K3), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E6.0, K3), Identity)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O248, E4.0, K7), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 50.00 msIt achieves 84.15% predicted accuracy with 44.93 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E4.0, K3), Identity)\n",
      "((O56, E6.0, K5), None)\n",
      "((O56, E6.0, K3), Identity)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O56, E3.0, K5), Identity)\n",
      "((O104, E4.0, K5), None)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O104, E3.0, K3), Identity)\n",
      "((O104, E3.0, K7), Identity)\n",
      "((O128, E6.0, K7), None)\n",
      "((O128, E4.0, K7), Identity)\n",
      "((O128, E6.0, K3), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E6.0, K3), Identity)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O248, E4.0, K7), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 55.00 msIt achieves 84.15% predicted accuracy with 44.93 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E4.0, K3), Identity)\n",
      "((O56, E6.0, K5), None)\n",
      "((O56, E6.0, K3), Identity)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O56, E3.0, K5), Identity)\n",
      "((O104, E4.0, K5), None)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O104, E3.0, K3), Identity)\n",
      "((O104, E3.0, K7), Identity)\n",
      "((O128, E6.0, K7), None)\n",
      "((O128, E4.0, K7), Identity)\n",
      "((O128, E6.0, K3), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E6.0, K3), Identity)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O248, E4.0, K7), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 60.00 msIt achieves 84.15% predicted accuracy with 44.93 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E4.0, K3), Identity)\n",
      "((O56, E6.0, K5), None)\n",
      "((O56, E6.0, K3), Identity)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O56, E3.0, K5), Identity)\n",
      "((O104, E4.0, K5), None)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O104, E3.0, K3), Identity)\n",
      "((O104, E3.0, K7), Identity)\n",
      "((O128, E6.0, K7), None)\n",
      "((O128, E4.0, K7), Identity)\n",
      "((O128, E6.0, K3), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E6.0, K3), Identity)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O248, E4.0, K7), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  76.1533989906311\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "latency_constraints = (15, 20, 25, 30, 35, 40, 45, 50, 55, 60)\n",
    "times[latency_constraints] = run_bottom_up_evolutionary_search(latency_constraints)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(15,): 29.050503969192505,\n",
       " (15, 20): 35.075645208358765,\n",
       " (15, 20, 25): 40.246270179748535,\n",
       " (15, 20, 25, 30): 52.753483057022095,\n",
       " (15, 20, 25, 30, 35): 56.32675790786743,\n",
       " (15, 20, 25, 30, 35, 40, 45, 50, 55, 60): 76.1533989906311}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "times"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Accuracy of Discovered Subnetworks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MobileNetV3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:19<00:00, 25.73it/s]\n",
      "Searching with note10 constraint (20): 100%|██████████| 100/100 [00:03<00:00, 25.13it/s]\n",
      "Searching with note10 constraint (25): 100%|██████████| 100/100 [00:03<00:00, 27.56it/s]\n",
      "Searching with note10 constraint (30): 100%|██████████| 100/100 [00:03<00:00, 29.03it/s]\n",
      "Searching with note10 constraint (35): 100%|██████████| 100/100 [00:03<00:00, 29.46it/s]\n",
      "Searching with note10 constraint (40): 100%|██████████| 100/100 [00:03<00:00, 30.36it/s]\n",
      "Searching with note10 constraint (45): 100%|██████████| 100/100 [00:03<00:00, 27.70it/s]\n",
      "Searching with note10 constraint (50): 100%|██████████| 100/100 [00:03<00:00, 29.86it/s]\n",
      "Searching with note10 constraint (55): 100%|██████████| 100/100 [00:03<00:00, 28.66it/s]\n",
      "Searching with note10 constraint (60): 100%|██████████| 100/100 [00:03<00:00, 28.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.63% predicted accuracy with 14.88 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E4.0, K3), Identity)\n",
      "((O96, E6.0, K5), Identity)\n",
      "(SE(O136, E4.0, K3), None)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O192, E4.0, K3), None)\n",
      "(SE(O192, E3.0, K3), Identity)\n",
      "(SE(O192, E3.0, K3), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.70% predicted accuracy with 19.92 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "((O96, E4.0, K7), None)\n",
      "((O96, E4.0, K3), Identity)\n",
      "((O96, E3.0, K3), Identity)\n",
      "(SE(O136, E4.0, K3), None)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O192, E4.0, K3), None)\n",
      "(SE(O192, E3.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 25.00 msIt achieves 81.97% predicted accuracy with 24.57 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "((O96, E3.0, K5), None)\n",
      "((O96, E4.0, K5), Identity)\n",
      "((O96, E3.0, K3), Identity)\n",
      "(SE(O136, E4.0, K7), None)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O192, E6.0, K7), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E4.0, K7), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 30.00 msIt achieves 82.88% predicted accuracy with 29.37 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E6.0, K5), Identity)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "((O96, E3.0, K5), None)\n",
      "((O96, E4.0, K5), Identity)\n",
      "((O96, E6.0, K3), Identity)\n",
      "(SE(O136, E3.0, K7), None)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O136, E6.0, K3), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O192, E6.0, K7), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E4.0, K7), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 35.00 msIt achieves 83.20% predicted accuracy with 34.60 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E6.0, K5), Identity)\n",
      "((O32, E3.0, K5), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E6.0, K5), Identity)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E4.0, K5), Identity)\n",
      "((O96, E6.0, K3), Identity)\n",
      "(SE(O136, E3.0, K7), None)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O136, E6.0, K3), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O192, E6.0, K7), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E4.0, K7), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 40.00 msIt achieves 83.93% predicted accuracy with 39.64 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E3.0, K5), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E6.0, K5), Identity)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "((O96, E3.0, K5), None)\n",
      "((O96, E4.0, K5), Identity)\n",
      "((O96, E6.0, K7), Identity)\n",
      "((O96, E3.0, K7), Identity)\n",
      "(SE(O136, E3.0, K7), None)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O136, E6.0, K3), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O192, E6.0, K7), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E4.0, K7), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 45.00 msIt achieves 84.06% predicted accuracy with 44.45 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E3.0, K5), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E6.0, K5), Identity)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "((O96, E3.0, K7), None)\n",
      "((O96, E4.0, K5), Identity)\n",
      "((O96, E6.0, K7), Identity)\n",
      "((O96, E3.0, K7), Identity)\n",
      "(SE(O136, E3.0, K7), None)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O136, E6.0, K3), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O192, E6.0, K7), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E4.0, K7), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 50.00 msIt achieves 84.09% predicted accuracy with 43.51 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E3.0, K5), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E6.0, K5), Identity)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "((O96, E3.0, K5), None)\n",
      "((O96, E4.0, K5), Identity)\n",
      "((O96, E6.0, K3), Identity)\n",
      "((O96, E3.0, K7), Identity)\n",
      "(SE(O136, E3.0, K7), None)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O136, E6.0, K3), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O192, E6.0, K7), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E4.0, K7), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 55.00 msIt achieves 84.19% predicted accuracy with 54.41 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E6.0, K7), Identity)\n",
      "((O32, E3.0, K5), Identity)\n",
      "(SE(O48, E4.0, K7), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E6.0, K5), Identity)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "((O96, E3.0, K5), None)\n",
      "((O96, E4.0, K5), Identity)\n",
      "((O96, E6.0, K7), Identity)\n",
      "((O96, E3.0, K7), Identity)\n",
      "(SE(O136, E3.0, K7), None)\n",
      "(SE(O136, E6.0, K5), Identity)\n",
      "(SE(O136, E6.0, K3), Identity)\n",
      "(SE(O136, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K7), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E4.0, K7), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 60.00 msIt achieves 84.41% predicted accuracy with 57.19 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K7), Identity)\n",
      "((O32, E3.0, K5), Identity)\n",
      "((O32, E6.0, K5), Identity)\n",
      "(SE(O48, E6.0, K7), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E6.0, K5), Identity)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "((O96, E3.0, K5), None)\n",
      "((O96, E4.0, K5), Identity)\n",
      "((O96, E6.0, K7), Identity)\n",
      "((O96, E3.0, K7), Identity)\n",
      "(SE(O136, E3.0, K7), None)\n",
      "(SE(O136, E6.0, K5), Identity)\n",
      "(SE(O136, E6.0, K3), Identity)\n",
      "(SE(O136, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K7), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E4.0, K7), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  62.214707136154175\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "62.214707136154175"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "latency_constraints = (15, 20, 25, 30, 35, 40, 45, 50, 55, 60)\n",
    "run_bottom_up_evolutionary_search(latency_constraints)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ResNet50D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:20<00:00, 24.29it/s]\n",
      "Searching with note10 constraint (20): 100%|██████████| 100/100 [00:04<00:00, 23.26it/s]\n",
      "Searching with note10 constraint (25): 100%|██████████| 100/100 [00:04<00:00, 24.99it/s]\n",
      "Searching with note10 constraint (30): 100%|██████████| 100/100 [00:03<00:00, 27.22it/s]\n",
      "Searching with note10 constraint (35): 100%|██████████| 100/100 [00:03<00:00, 29.58it/s]\n",
      "Searching with note10 constraint (40): 100%|██████████| 100/100 [00:03<00:00, 27.78it/s]\n",
      "Searching with note10 constraint (45): 100%|██████████| 100/100 [00:03<00:00, 28.93it/s]\n",
      "Searching with note10 constraint (50): 100%|██████████| 100/100 [00:03<00:00, 26.61it/s]\n",
      "Searching with note10 constraint (55): 100%|██████████| 100/100 [00:03<00:00, 27.75it/s]\n",
      "Searching with note10 constraint (60): 100%|██████████| 100/100 [00:03<00:00, 29.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.50% predicted accuracy with 14.94 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.70% predicted accuracy with 19.92 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 25.00 msIt achieves 81.99% predicted accuracy with 24.96 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 30.00 msIt achieves 82.76% predicted accuracy with 29.92 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 35.00 msIt achieves 83.28% predicted accuracy with 34.01 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 40.00 msIt achieves 83.72% predicted accuracy with 39.68 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 45.00 msIt achieves 83.96% predicted accuracy with 44.11 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "(DyConv(O32, K3, S1), Identity)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 50.00 msIt achieves 84.11% predicted accuracy with 49.94 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->768->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 55.00 msIt achieves 84.48% predicted accuracy with 54.77 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Found best architecture on note10 with latency <= 60.00 msIt achieves 84.52% predicted accuracy with 55.59 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "DyConv(O32, K3, S2)\n",
      "DyConv(O64, K3, S1)\n",
      "max_pooling(ks=3, stride=2)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->768->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->1536->256_S1, Identity)\n",
      "(3x3_BottleneckConv_in->2048->512_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->2048->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->3072->512_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->6144->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->4096->1024_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S2, avgpool_conv)\n",
      "(3x3_BottleneckConv_in->8192->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "(3x3_BottleneckConv_in->12288->2048_S1, Identity)\n",
      "MyGlobalAvgPool2d(keep_dim=False)\n",
      "DyLinear(2048, 1000)\n",
      "Time taken for latency constraint:  63.87831711769104\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "63.87831711769104"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "latency_constraints = (15, 20, 25, 30, 35, 40, 45, 50, 55, 60)\n",
    "run_bottom_up_evolutionary_search(latency_constraints)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ProxylessNAS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:21<00:00, 23.30it/s]\n",
      "Searching with note10 constraint (20): 100%|██████████| 100/100 [00:04<00:00, 23.21it/s]\n",
      "Searching with note10 constraint (25): 100%|██████████| 100/100 [00:03<00:00, 25.12it/s]\n",
      "Searching with note10 constraint (30): 100%|██████████| 100/100 [00:04<00:00, 24.84it/s]\n",
      "Searching with note10 constraint (35): 100%|██████████| 100/100 [00:04<00:00, 24.88it/s]\n",
      "Searching with note10 constraint (40): 100%|██████████| 100/100 [00:03<00:00, 27.07it/s]\n",
      "Searching with note10 constraint (45): 100%|██████████| 100/100 [00:03<00:00, 29.76it/s]\n",
      "Searching with note10 constraint (50): 100%|██████████| 100/100 [00:03<00:00, 26.94it/s]\n",
      "Searching with note10 constraint (55): 100%|██████████| 100/100 [00:03<00:00, 27.56it/s]\n",
      "Searching with note10 constraint (60): 100%|██████████| 100/100 [00:03<00:00, 28.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.68% predicted accuracy with 14.96 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K5), None)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O104, E4.0, K5), None)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O128, E4.0, K3), None)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O128, E4.0, K5), Identity)\n",
      "((O248, E4.0, K3), None)\n",
      "((O248, E4.0, K3), Identity)\n",
      "((O248, E4.0, K3), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.64% predicted accuracy with 19.97 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K3), None)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O56, E3.0, K5), Identity)\n",
      "((O104, E4.0, K5), None)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O128, E4.0, K3), None)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O128, E4.0, K5), Identity)\n",
      "((O248, E4.0, K3), None)\n",
      "((O248, E4.0, K3), Identity)\n",
      "((O248, E4.0, K3), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 25.00 msIt achieves 81.48% predicted accuracy with 24.95 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K3), None)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O104, E4.0, K5), None)\n",
      "((O104, E3.0, K7), Identity)\n",
      "((O104, E4.0, K5), Identity)\n",
      "((O128, E4.0, K5), None)\n",
      "((O128, E4.0, K5), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E4.0, K3), Identity)\n",
      "((O248, E4.0, K5), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 30.00 msIt achieves 82.24% predicted accuracy with 29.97 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E6.0, K3), Identity)\n",
      "((O56, E4.0, K3), None)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O104, E4.0, K7), None)\n",
      "((O104, E3.0, K7), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O104, E4.0, K3), Identity)\n",
      "((O128, E4.0, K5), None)\n",
      "((O128, E4.0, K5), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O128, E4.0, K5), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E4.0, K3), Identity)\n",
      "((O248, E4.0, K5), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 35.00 msIt achieves 82.95% predicted accuracy with 34.84 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K3), None)\n",
      "((O56, E4.0, K5), Identity)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O104, E4.0, K7), None)\n",
      "((O104, E3.0, K7), Identity)\n",
      "((O104, E3.0, K5), Identity)\n",
      "((O104, E4.0, K7), Identity)\n",
      "((O128, E4.0, K5), None)\n",
      "((O128, E4.0, K5), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O128, E4.0, K5), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 40.00 msIt achieves 83.76% predicted accuracy with 39.41 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K3), None)\n",
      "((O56, E4.0, K5), Identity)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O104, E4.0, K7), None)\n",
      "((O104, E3.0, K7), Identity)\n",
      "((O104, E6.0, K7), Identity)\n",
      "((O104, E6.0, K7), Identity)\n",
      "((O128, E6.0, K3), None)\n",
      "((O128, E4.0, K5), Identity)\n",
      "((O128, E4.0, K3), Identity)\n",
      "((O128, E4.0, K5), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O248, E6.0, K3), Identity)\n",
      "((O248, E3.0, K7), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 45.00 msIt achieves 83.97% predicted accuracy with 44.41 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O56, E4.0, K3), None)\n",
      "((O56, E6.0, K5), Identity)\n",
      "((O56, E4.0, K7), Identity)\n",
      "((O104, E4.0, K7), None)\n",
      "((O104, E3.0, K7), Identity)\n",
      "((O104, E6.0, K7), Identity)\n",
      "((O104, E6.0, K7), Identity)\n",
      "((O128, E6.0, K5), None)\n",
      "((O128, E4.0, K5), Identity)\n",
      "((O128, E6.0, K3), Identity)\n",
      "((O128, E6.0, K5), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O248, E3.0, K5), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 50.00 msIt achieves 84.45% predicted accuracy with 49.85 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E4.0, K3), Identity)\n",
      "((O32, E3.0, K7), Identity)\n",
      "((O56, E4.0, K3), None)\n",
      "((O56, E6.0, K5), Identity)\n",
      "((O56, E4.0, K3), Identity)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O104, E4.0, K7), None)\n",
      "((O104, E3.0, K7), Identity)\n",
      "((O104, E6.0, K5), Identity)\n",
      "((O104, E6.0, K7), Identity)\n",
      "((O128, E6.0, K7), None)\n",
      "((O128, E4.0, K7), Identity)\n",
      "((O128, E6.0, K3), Identity)\n",
      "((O128, E6.0, K5), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O248, E3.0, K5), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 55.00 msIt achieves 84.68% predicted accuracy with 54.06 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E3.0, K7), Identity)\n",
      "((O32, E3.0, K5), Identity)\n",
      "((O56, E4.0, K3), None)\n",
      "((O56, E6.0, K5), Identity)\n",
      "((O56, E4.0, K7), Identity)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O104, E4.0, K7), None)\n",
      "((O104, E3.0, K7), Identity)\n",
      "((O104, E6.0, K7), Identity)\n",
      "((O104, E6.0, K7), Identity)\n",
      "((O128, E6.0, K7), None)\n",
      "((O128, E4.0, K7), Identity)\n",
      "((O128, E6.0, K3), Identity)\n",
      "((O128, E6.0, K5), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E6.0, K7), Identity)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O248, E3.0, K5), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Found best architecture on note10 with latency <= 60.00 msIt achieves 84.68% predicted accuracy with 54.06 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O40_RELU6_BN\n",
      "(3x3_MBConv1_RELU6_O24_BN, None)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E3.0, K7), Identity)\n",
      "((O32, E3.0, K5), Identity)\n",
      "((O56, E4.0, K3), None)\n",
      "((O56, E6.0, K5), Identity)\n",
      "((O56, E4.0, K7), Identity)\n",
      "((O56, E3.0, K3), Identity)\n",
      "((O104, E4.0, K7), None)\n",
      "((O104, E3.0, K7), Identity)\n",
      "((O104, E6.0, K7), Identity)\n",
      "((O104, E6.0, K7), Identity)\n",
      "((O128, E6.0, K7), None)\n",
      "((O128, E4.0, K7), Identity)\n",
      "((O128, E6.0, K3), Identity)\n",
      "((O128, E6.0, K5), Identity)\n",
      "((O248, E6.0, K3), None)\n",
      "((O248, E6.0, K7), Identity)\n",
      "((O248, E6.0, K5), Identity)\n",
      "((O248, E3.0, K5), Identity)\n",
      "((O416, E6.0, K7), None)\n",
      "1x1_Conv_O1664_RELU6_BN\n",
      "1664x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  63.48898005485535\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "63.48898005485535"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "latency_constraints = (15, 20, 25, 30, 35, 40, 45, 50, 55, 60)\n",
    "run_bottom_up_evolutionary_search(latency_constraints)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cost of finding a single latency constraint subnetwork"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "times = {}\n",
    "latency_constraints = (15, 20, 25, 30, 35, 40, 45, 50, 55, 60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Searching with note10 constraint (15):   0%|          | 0/500 [00:00<?, ?it/s]\u001b[A\n",
      "Searching with note10 constraint (15):   1%|          | 3/500 [00:00<00:19, 24.94it/s]\u001b[A\n",
      "Searching with note10 constraint (15):   1%|          | 6/500 [00:00<00:20, 24.54it/s]\u001b[A\n",
      "Searching with note10 constraint (15):   2%|▏         | 9/500 [00:00<00:19, 24.57it/s]\u001b[A\n",
      "Searching with note10 constraint (15):   2%|▏         | 12/500 [00:00<00:19, 24.93it/s]\u001b[A\n",
      "Searching with note10 constraint (15):   3%|▎         | 15/500 [00:00<00:19, 25.10it/s]\u001b[A\n",
      "Searching with note10 constraint (15):   4%|▎         | 18/500 [00:00<00:19, 25.07it/s]\u001b[A\n",
      "Searching with note10 constraint (15):   4%|▍         | 21/500 [00:00<00:18, 25.44it/s]\u001b[A\n",
      "Searching with note10 constraint (15):   5%|▍         | 24/500 [00:00<00:18, 25.29it/s]\u001b[A\n",
      "Searching with note10 constraint (15):   5%|▌         | 27/500 [00:01<00:18, 25.36it/s]\u001b[A\n",
      "Searching with note10 constraint (15):   6%|▌         | 30/500 [00:01<00:18, 25.71it/s]\u001b[A\n",
      "Searching with note10 constraint (15):   7%|▋         | 33/500 [00:01<00:18, 25.73it/s]\u001b[A\n",
      "Searching with note10 constraint (15):   7%|▋         | 36/500 [00:01<00:18, 25.69it/s]\u001b[A\n",
      "Searching with note10 constraint (15):   8%|▊         | 39/500 [00:01<00:18, 25.12it/s]\u001b[A\n",
      "Searching with note10 constraint (15):   8%|▊         | 42/500 [00:01<00:18, 24.48it/s]\u001b[A\n",
      "Searching with note10 constraint (15):   9%|▉         | 45/500 [00:01<00:18, 24.10it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  10%|▉         | 48/500 [00:01<00:18, 24.64it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  10%|█         | 51/500 [00:02<00:18, 24.72it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  11%|█         | 54/500 [00:02<00:18, 24.69it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  11%|█▏        | 57/500 [00:02<00:17, 24.80it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  12%|█▏        | 60/500 [00:02<00:17, 25.20it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  13%|█▎        | 63/500 [00:02<00:16, 25.73it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  13%|█▎        | 66/500 [00:02<00:17, 24.31it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  14%|█▍        | 69/500 [00:02<00:18, 23.07it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  14%|█▍        | 72/500 [00:02<00:19, 22.28it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  15%|█▌        | 75/500 [00:03<00:18, 22.46it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  16%|█▌        | 78/500 [00:03<00:18, 22.95it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  16%|█▌        | 81/500 [00:03<00:18, 23.25it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  17%|█▋        | 84/500 [00:03<00:18, 23.03it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  17%|█▋        | 87/500 [00:03<00:17, 23.08it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  18%|█▊        | 90/500 [00:03<00:17, 23.02it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  19%|█▊        | 93/500 [00:03<00:17, 23.81it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  19%|█▉        | 96/500 [00:03<00:16, 24.11it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  20%|█▉        | 99/500 [00:04<00:16, 24.31it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  20%|██        | 102/500 [00:04<00:17, 22.17it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  21%|██        | 105/500 [00:04<00:17, 21.98it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  22%|██▏       | 108/500 [00:04<00:18, 21.55it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  22%|██▏       | 111/500 [00:04<00:17, 21.77it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  23%|██▎       | 114/500 [00:04<00:17, 22.13it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  23%|██▎       | 117/500 [00:04<00:16, 22.96it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  24%|██▍       | 120/500 [00:05<00:16, 23.68it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  25%|██▍       | 123/500 [00:05<00:15, 23.78it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  25%|██▌       | 126/500 [00:05<00:15, 23.86it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  26%|██▌       | 129/500 [00:05<00:16, 22.90it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  26%|██▋       | 132/500 [00:05<00:16, 22.91it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  27%|██▋       | 135/500 [00:05<00:15, 23.56it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  28%|██▊       | 138/500 [00:05<00:15, 23.98it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  28%|██▊       | 141/500 [00:05<00:15, 23.69it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  29%|██▉       | 144/500 [00:06<00:15, 23.24it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  29%|██▉       | 147/500 [00:06<00:15, 23.11it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  30%|███       | 150/500 [00:06<00:15, 22.95it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  31%|███       | 153/500 [00:06<00:15, 23.02it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  31%|███       | 156/500 [00:06<00:14, 23.58it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  32%|███▏      | 159/500 [00:06<00:13, 24.38it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  32%|███▏      | 162/500 [00:06<00:13, 24.80it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  33%|███▎      | 165/500 [00:06<00:13, 24.82it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  34%|███▎      | 168/500 [00:07<00:13, 24.64it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  34%|███▍      | 171/500 [00:07<00:13, 24.19it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  35%|███▍      | 174/500 [00:07<00:13, 23.51it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  35%|███▌      | 177/500 [00:07<00:13, 23.38it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  36%|███▌      | 180/500 [00:07<00:13, 23.80it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  37%|███▋      | 183/500 [00:07<00:12, 24.41it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  37%|███▋      | 186/500 [00:07<00:13, 23.90it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  38%|███▊      | 189/500 [00:07<00:12, 24.18it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  38%|███▊      | 192/500 [00:08<00:13, 22.13it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  39%|███▉      | 195/500 [00:08<00:13, 22.07it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  40%|███▉      | 198/500 [00:08<00:13, 22.51it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  40%|████      | 201/500 [00:08<00:13, 22.91it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  41%|████      | 204/500 [00:08<00:13, 21.57it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  41%|████▏     | 207/500 [00:08<00:13, 22.05it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  42%|████▏     | 210/500 [00:08<00:12, 22.42it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  43%|████▎     | 213/500 [00:09<00:12, 22.48it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  43%|████▎     | 216/500 [00:09<00:12, 22.85it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  44%|████▍     | 219/500 [00:09<00:12, 22.06it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  44%|████▍     | 222/500 [00:09<00:14, 19.72it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  45%|████▌     | 225/500 [00:09<00:13, 19.65it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  46%|████▌     | 228/500 [00:09<00:13, 20.67it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  46%|████▌     | 231/500 [00:09<00:13, 20.32it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  47%|████▋     | 234/500 [00:10<00:12, 20.63it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  47%|████▋     | 237/500 [00:10<00:12, 21.37it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  48%|████▊     | 240/500 [00:10<00:11, 21.70it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  49%|████▊     | 243/500 [00:10<00:12, 21.29it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  49%|████▉     | 246/500 [00:10<00:11, 22.01it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  50%|████▉     | 249/500 [00:10<00:11, 21.80it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  50%|█████     | 252/500 [00:10<00:11, 21.99it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  51%|█████     | 255/500 [00:10<00:10, 22.63it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  52%|█████▏    | 258/500 [00:11<00:10, 22.36it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  52%|█████▏    | 261/500 [00:11<00:10, 22.11it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  53%|█████▎    | 264/500 [00:11<00:10, 22.45it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  53%|█████▎    | 267/500 [00:11<00:10, 23.01it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  54%|█████▍    | 270/500 [00:11<00:10, 22.99it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  55%|█████▍    | 273/500 [00:11<00:09, 22.97it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  55%|█████▌    | 276/500 [00:11<00:09, 23.01it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  56%|█████▌    | 279/500 [00:12<00:09, 23.03it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  56%|█████▋    | 282/500 [00:12<00:09, 23.33it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  57%|█████▋    | 285/500 [00:12<00:09, 23.36it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  58%|█████▊    | 288/500 [00:12<00:09, 22.94it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  58%|█████▊    | 291/500 [00:12<00:09, 23.04it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  59%|█████▉    | 294/500 [00:12<00:09, 22.56it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  59%|█████▉    | 297/500 [00:12<00:09, 22.42it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  60%|██████    | 300/500 [00:12<00:09, 21.57it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  61%|██████    | 303/500 [00:13<00:10, 18.17it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  61%|██████    | 305/500 [00:13<00:12, 16.11it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  61%|██████▏   | 307/500 [00:13<00:11, 16.88it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  62%|██████▏   | 310/500 [00:13<00:10, 18.19it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  63%|██████▎   | 313/500 [00:13<00:09, 19.47it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  63%|██████▎   | 316/500 [00:13<00:08, 20.52it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  64%|██████▍   | 319/500 [00:14<00:08, 20.78it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  64%|██████▍   | 322/500 [00:14<00:10, 17.16it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  65%|██████▍   | 324/500 [00:14<00:11, 14.70it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  65%|██████▌   | 326/500 [00:14<00:11, 15.33it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  66%|██████▌   | 328/500 [00:14<00:10, 15.80it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  66%|██████▌   | 330/500 [00:14<00:10, 16.75it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  67%|██████▋   | 333/500 [00:14<00:09, 17.84it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  67%|██████▋   | 335/500 [00:15<00:09, 16.89it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  67%|██████▋   | 337/500 [00:15<00:11, 13.83it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  68%|██████▊   | 339/500 [00:15<00:12, 13.39it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  68%|██████▊   | 341/500 [00:15<00:11, 14.03it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  69%|██████▊   | 343/500 [00:15<00:10, 15.08it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  69%|██████▉   | 345/500 [00:15<00:09, 15.91it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  69%|██████▉   | 347/500 [00:15<00:09, 16.38it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  70%|██████▉   | 349/500 [00:15<00:08, 16.98it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  70%|███████   | 351/500 [00:16<00:08, 17.74it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  71%|███████   | 354/500 [00:16<00:07, 18.44it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  71%|███████▏  | 357/500 [00:16<00:07, 19.53it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  72%|███████▏  | 360/500 [00:16<00:07, 19.96it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  73%|███████▎  | 363/500 [00:16<00:06, 20.25it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  73%|███████▎  | 366/500 [00:16<00:06, 20.24it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  74%|███████▍  | 369/500 [00:16<00:06, 20.26it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  74%|███████▍  | 372/500 [00:17<00:06, 20.78it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  75%|███████▌  | 375/500 [00:17<00:06, 20.75it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  76%|███████▌  | 378/500 [00:17<00:05, 20.66it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  76%|███████▌  | 381/500 [00:17<00:05, 20.35it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  77%|███████▋  | 384/500 [00:17<00:05, 20.24it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  77%|███████▋  | 387/500 [00:17<00:05, 20.53it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  78%|███████▊  | 390/500 [00:17<00:05, 21.06it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  79%|███████▊  | 393/500 [00:18<00:05, 20.98it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  79%|███████▉  | 396/500 [00:18<00:04, 20.89it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  80%|███████▉  | 399/500 [00:18<00:04, 21.33it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  80%|████████  | 402/500 [00:18<00:04, 21.34it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  81%|████████  | 405/500 [00:18<00:04, 21.20it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  82%|████████▏ | 408/500 [00:18<00:04, 21.35it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  82%|████████▏ | 411/500 [00:18<00:04, 21.22it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  83%|████████▎ | 414/500 [00:19<00:04, 20.90it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  83%|████████▎ | 417/500 [00:19<00:03, 20.81it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  84%|████████▍ | 420/500 [00:19<00:03, 20.72it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  85%|████████▍ | 423/500 [00:19<00:03, 19.65it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  85%|████████▌ | 425/500 [00:19<00:03, 19.53it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  85%|████████▌ | 427/500 [00:19<00:03, 18.68it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  86%|████████▌ | 430/500 [00:19<00:03, 19.36it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  86%|████████▋ | 432/500 [00:20<00:03, 19.03it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  87%|████████▋ | 435/500 [00:20<00:03, 19.63it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  87%|████████▋ | 437/500 [00:20<00:03, 18.97it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  88%|████████▊ | 440/500 [00:20<00:03, 19.53it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  88%|████████▊ | 442/500 [00:20<00:03, 19.06it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  89%|████████▉ | 444/500 [00:20<00:02, 19.10it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  89%|████████▉ | 446/500 [00:20<00:02, 19.20it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  90%|████████▉ | 449/500 [00:20<00:02, 19.82it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  90%|█████████ | 452/500 [00:21<00:02, 20.34it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  91%|█████████ | 455/500 [00:21<00:02, 20.46it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  92%|█████████▏| 458/500 [00:21<00:02, 20.24it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  92%|█████████▏| 461/500 [00:21<00:01, 20.46it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  93%|█████████▎| 464/500 [00:21<00:01, 20.40it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  93%|█████████▎| 467/500 [00:21<00:01, 20.41it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  94%|█████████▍| 470/500 [00:21<00:01, 21.21it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  95%|█████████▍| 473/500 [00:22<00:01, 20.93it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  95%|█████████▌| 476/500 [00:22<00:01, 20.61it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  96%|█████████▌| 479/500 [00:22<00:01, 20.81it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  96%|█████████▋| 482/500 [00:22<00:00, 20.91it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  97%|█████████▋| 485/500 [00:22<00:00, 20.97it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  98%|█████████▊| 488/500 [00:22<00:00, 20.62it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  98%|█████████▊| 491/500 [00:22<00:00, 20.64it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  99%|█████████▉| 494/500 [00:23<00:00, 20.83it/s]\u001b[A\n",
      "Searching with note10 constraint (15):  99%|█████████▉| 497/500 [00:23<00:00, 20.00it/s]\u001b[A\n",
      "Searching with note10 constraint (15): 100%|██████████| 500/500 [00:23<00:00, 21.41it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 15.00 msIt achieves 78.47% predicted accuracy with 14.99 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E4.0, K3), Identity)\n",
      "(SE(O48, E3.0, K3), None)\n",
      "(SE(O48, E3.0, K5), Identity)\n",
      "((O96, E6.0, K5), None)\n",
      "((O96, E3.0, K3), Identity)\n",
      "((O96, E4.0, K5), Identity)\n",
      "((O96, E4.0, K3), Identity)\n",
      "(SE(O136, E4.0, K3), None)\n",
      "(SE(O136, E4.0, K7), Identity)\n",
      "(SE(O136, E3.0, K3), Identity)\n",
      "(SE(O192, E3.0, K3), None)\n",
      "(SE(O192, E4.0, K3), Identity)\n",
      "(SE(O192, E4.0, K3), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  31.404531002044678\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Searching with note10 constraint (20):   0%|          | 0/500 [00:00<?, ?it/s]\u001b[A\n",
      "Searching with note10 constraint (20):   1%|          | 3/500 [00:00<00:20, 24.25it/s]\u001b[A\n",
      "Searching with note10 constraint (20):   1%|          | 6/500 [00:00<00:20, 23.65it/s]\u001b[A\n",
      "Searching with note10 constraint (20):   2%|▏         | 9/500 [00:00<00:21, 22.84it/s]\u001b[A\n",
      "Searching with note10 constraint (20):   2%|▏         | 12/500 [00:00<00:22, 22.03it/s]\u001b[A\n",
      "Searching with note10 constraint (20):   3%|▎         | 14/500 [00:00<00:23, 20.91it/s]\u001b[A\n",
      "Searching with note10 constraint (20):   3%|▎         | 16/500 [00:00<00:23, 20.57it/s]\u001b[A\n",
      "Searching with note10 constraint (20):   4%|▎         | 18/500 [00:00<00:23, 20.25it/s]\u001b[A\n",
      "Searching with note10 constraint (20):   4%|▍         | 20/500 [00:00<00:24, 19.83it/s]\u001b[A\n",
      "Searching with note10 constraint (20):   5%|▍         | 23/500 [00:01<00:23, 20.56it/s]\u001b[A\n",
      "Searching with note10 constraint (20):   5%|▌         | 26/500 [00:01<00:22, 20.73it/s]\u001b[A\n",
      "Searching with note10 constraint (20):   6%|▌         | 29/500 [00:01<00:22, 21.10it/s]\u001b[A\n",
      "Searching with note10 constraint (20):   6%|▋         | 32/500 [00:01<00:22, 20.38it/s]\u001b[A\n",
      "Searching with note10 constraint (20):   7%|▋         | 35/500 [00:01<00:22, 20.59it/s]\u001b[A\n",
      "Searching with note10 constraint (20):   8%|▊         | 38/500 [00:01<00:22, 20.83it/s]\u001b[A\n",
      "Searching with note10 constraint (20):   8%|▊         | 41/500 [00:01<00:21, 20.87it/s]\u001b[A\n",
      "Searching with note10 constraint (20):   9%|▉         | 44/500 [00:02<00:21, 21.18it/s]\u001b[A\n",
      "Searching with note10 constraint (20):   9%|▉         | 47/500 [00:02<00:20, 21.63it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  10%|█         | 50/500 [00:02<00:20, 21.62it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  11%|█         | 53/500 [00:02<00:20, 21.57it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  11%|█         | 56/500 [00:02<00:20, 21.69it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  12%|█▏        | 59/500 [00:02<00:19, 22.05it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  12%|█▏        | 62/500 [00:02<00:19, 22.48it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  13%|█▎        | 65/500 [00:03<00:19, 22.04it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  14%|█▎        | 68/500 [00:03<00:19, 21.82it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  14%|█▍        | 71/500 [00:03<00:19, 21.78it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  15%|█▍        | 74/500 [00:03<00:19, 21.98it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  15%|█▌        | 77/500 [00:03<00:19, 22.10it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  16%|█▌        | 80/500 [00:03<00:18, 22.21it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  17%|█▋        | 83/500 [00:03<00:18, 22.36it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  17%|█▋        | 86/500 [00:04<00:18, 21.89it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  18%|█▊        | 89/500 [00:04<00:19, 21.05it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  18%|█▊        | 92/500 [00:04<00:18, 21.79it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  19%|█▉        | 95/500 [00:04<00:18, 21.65it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  20%|█▉        | 98/500 [00:04<00:19, 20.69it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  20%|██        | 101/500 [00:04<00:19, 20.48it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  21%|██        | 104/500 [00:04<00:18, 21.02it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  21%|██▏       | 107/500 [00:05<00:19, 20.65it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  22%|██▏       | 110/500 [00:05<00:18, 20.78it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  23%|██▎       | 113/500 [00:05<00:18, 21.02it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  23%|██▎       | 116/500 [00:05<00:18, 21.13it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  24%|██▍       | 119/500 [00:05<00:18, 20.81it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  24%|██▍       | 122/500 [00:05<00:18, 20.69it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  25%|██▌       | 125/500 [00:05<00:18, 20.10it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  26%|██▌       | 128/500 [00:06<00:18, 19.88it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  26%|██▌       | 131/500 [00:06<00:18, 20.21it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  27%|██▋       | 134/500 [00:06<00:18, 20.18it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  27%|██▋       | 137/500 [00:06<00:18, 20.03it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  28%|██▊       | 140/500 [00:06<00:18, 19.87it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  29%|██▊       | 143/500 [00:06<00:17, 20.18it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  29%|██▉       | 146/500 [00:06<00:17, 20.23it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  30%|██▉       | 149/500 [00:07<00:17, 20.19it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  30%|███       | 152/500 [00:07<00:17, 20.39it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  31%|███       | 155/500 [00:07<00:16, 20.33it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  32%|███▏      | 158/500 [00:07<00:16, 20.86it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  32%|███▏      | 161/500 [00:07<00:16, 20.64it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  33%|███▎      | 164/500 [00:07<00:16, 20.62it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  33%|███▎      | 167/500 [00:07<00:16, 20.58it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  34%|███▍      | 170/500 [00:08<00:15, 20.74it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  35%|███▍      | 173/500 [00:08<00:15, 20.48it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  35%|███▌      | 176/500 [00:08<00:15, 20.37it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  36%|███▌      | 179/500 [00:08<00:15, 20.28it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  36%|███▋      | 182/500 [00:08<00:16, 19.87it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  37%|███▋      | 184/500 [00:08<00:15, 19.78it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  37%|███▋      | 186/500 [00:08<00:16, 19.48it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  38%|███▊      | 189/500 [00:09<00:15, 19.85it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  38%|███▊      | 191/500 [00:09<00:15, 19.85it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  39%|███▊      | 193/500 [00:09<00:15, 19.61it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  39%|███▉      | 196/500 [00:09<00:15, 20.19it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  40%|███▉      | 199/500 [00:09<00:15, 19.75it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  40%|████      | 201/500 [00:09<00:15, 19.22it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  41%|████      | 203/500 [00:09<00:15, 19.32it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  41%|████      | 205/500 [00:09<00:15, 19.08it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  42%|████▏     | 208/500 [00:10<00:14, 19.72it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  42%|████▏     | 211/500 [00:10<00:14, 20.14it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  43%|████▎     | 214/500 [00:10<00:13, 20.50it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  43%|████▎     | 217/500 [00:10<00:13, 20.37it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  44%|████▍     | 220/500 [00:10<00:14, 19.02it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  44%|████▍     | 222/500 [00:10<00:14, 18.83it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  45%|████▌     | 225/500 [00:10<00:14, 19.54it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  46%|████▌     | 228/500 [00:11<00:13, 19.92it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  46%|████▌     | 231/500 [00:11<00:13, 20.18it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  47%|████▋     | 234/500 [00:11<00:13, 19.96it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  47%|████▋     | 237/500 [00:11<00:13, 19.03it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  48%|████▊     | 239/500 [00:11<00:14, 18.04it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  48%|████▊     | 241/500 [00:11<00:14, 18.33it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  49%|████▊     | 243/500 [00:11<00:13, 18.37it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  49%|████▉     | 245/500 [00:11<00:14, 17.92it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  49%|████▉     | 247/500 [00:12<00:14, 17.44it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  50%|████▉     | 249/500 [00:12<00:15, 16.73it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  50%|█████     | 251/500 [00:12<00:14, 16.91it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  51%|█████     | 253/500 [00:12<00:14, 17.07it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  51%|█████     | 255/500 [00:12<00:14, 16.89it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  51%|█████▏    | 257/500 [00:12<00:14, 17.13it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  52%|█████▏    | 259/500 [00:12<00:13, 17.70it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  52%|█████▏    | 261/500 [00:12<00:13, 18.02it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  53%|█████▎    | 264/500 [00:13<00:12, 18.53it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  53%|█████▎    | 266/500 [00:13<00:12, 18.82it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  54%|█████▍    | 269/500 [00:13<00:11, 19.39it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  54%|█████▍    | 271/500 [00:13<00:11, 19.40it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  55%|█████▍    | 273/500 [00:13<00:11, 19.21it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  55%|█████▌    | 275/500 [00:13<00:11, 19.33it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  56%|█████▌    | 278/500 [00:13<00:11, 19.85it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  56%|█████▌    | 281/500 [00:13<00:10, 20.26it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  57%|█████▋    | 284/500 [00:14<00:10, 20.51it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  57%|█████▋    | 287/500 [00:14<00:10, 20.27it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  58%|█████▊    | 290/500 [00:14<00:10, 20.63it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  59%|█████▊    | 293/500 [00:14<00:10, 20.09it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  59%|█████▉    | 296/500 [00:14<00:10, 19.79it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  60%|█████▉    | 298/500 [00:14<00:10, 19.53it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  60%|██████    | 301/500 [00:14<00:09, 20.17it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  61%|██████    | 304/500 [00:15<00:10, 19.21it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  61%|██████    | 306/500 [00:15<00:10, 18.86it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  62%|██████▏   | 308/500 [00:15<00:10, 18.62it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  62%|██████▏   | 310/500 [00:15<00:10, 18.05it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  62%|██████▏   | 312/500 [00:15<00:10, 17.32it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  63%|██████▎   | 314/500 [00:15<00:10, 17.17it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  63%|██████▎   | 316/500 [00:15<00:10, 17.79it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  64%|██████▎   | 318/500 [00:15<00:10, 18.02it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  64%|██████▍   | 320/500 [00:15<00:09, 18.21it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  64%|██████▍   | 322/500 [00:16<00:09, 18.49it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  65%|██████▍   | 324/500 [00:16<00:09, 18.79it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  65%|██████▌   | 326/500 [00:16<00:09, 18.64it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  66%|██████▌   | 329/500 [00:16<00:09, 18.82it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  66%|██████▌   | 331/500 [00:16<00:09, 18.58it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  67%|██████▋   | 333/500 [00:16<00:08, 18.69it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  67%|██████▋   | 335/500 [00:16<00:09, 18.12it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  67%|██████▋   | 337/500 [00:16<00:09, 17.47it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  68%|██████▊   | 339/500 [00:16<00:09, 17.06it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  68%|██████▊   | 341/500 [00:17<00:09, 16.74it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  69%|██████▊   | 343/500 [00:17<00:09, 16.61it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  69%|██████▉   | 345/500 [00:17<00:08, 17.31it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  69%|██████▉   | 347/500 [00:17<00:08, 17.73it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  70%|██████▉   | 349/500 [00:17<00:08, 17.97it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  70%|███████   | 351/500 [00:17<00:08, 17.31it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  71%|███████   | 353/500 [00:17<00:08, 17.95it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  71%|███████   | 355/500 [00:17<00:07, 18.32it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  71%|███████▏  | 357/500 [00:17<00:07, 18.72it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  72%|███████▏  | 359/500 [00:18<00:07, 18.94it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  72%|███████▏  | 361/500 [00:18<00:07, 19.14it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  73%|███████▎  | 363/500 [00:18<00:07, 18.83it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  73%|███████▎  | 365/500 [00:18<00:07, 18.57it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  73%|███████▎  | 367/500 [00:18<00:07, 17.98it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  74%|███████▍  | 369/500 [00:18<00:07, 17.56it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  74%|███████▍  | 371/500 [00:18<00:07, 17.42it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  75%|███████▍  | 373/500 [00:18<00:07, 17.09it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  75%|███████▌  | 375/500 [00:19<00:07, 16.52it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  75%|███████▌  | 377/500 [00:19<00:07, 17.27it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  76%|███████▌  | 379/500 [00:19<00:06, 17.92it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  76%|███████▌  | 381/500 [00:19<00:06, 18.43it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  77%|███████▋  | 383/500 [00:19<00:06, 18.80it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  77%|███████▋  | 386/500 [00:19<00:05, 19.51it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  78%|███████▊  | 389/500 [00:19<00:05, 20.18it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  78%|███████▊  | 392/500 [00:19<00:05, 20.52it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  79%|███████▉  | 395/500 [00:20<00:05, 19.89it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  80%|███████▉  | 398/500 [00:20<00:05, 19.80it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  80%|████████  | 401/500 [00:20<00:04, 20.25it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  81%|████████  | 404/500 [00:20<00:04, 20.14it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  81%|████████▏ | 407/500 [00:20<00:04, 19.27it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  82%|████████▏ | 410/500 [00:20<00:04, 19.88it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  83%|████████▎ | 413/500 [00:20<00:04, 20.15it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  83%|████████▎ | 416/500 [00:21<00:04, 19.95it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  84%|████████▍ | 419/500 [00:21<00:04, 19.96it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  84%|████████▍ | 422/500 [00:21<00:04, 18.24it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  85%|████████▍ | 424/500 [00:21<00:04, 17.45it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  85%|████████▌ | 426/500 [00:21<00:04, 17.84it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  86%|████████▌ | 428/500 [00:21<00:04, 17.76it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  86%|████████▌ | 430/500 [00:21<00:03, 17.82it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  87%|████████▋ | 433/500 [00:22<00:03, 18.62it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  87%|████████▋ | 436/500 [00:22<00:03, 19.34it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  88%|████████▊ | 439/500 [00:22<00:03, 20.04it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  88%|████████▊ | 442/500 [00:22<00:02, 20.59it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  89%|████████▉ | 445/500 [00:22<00:02, 19.91it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  90%|████████▉ | 448/500 [00:22<00:02, 19.94it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  90%|█████████ | 451/500 [00:22<00:02, 20.32it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  91%|█████████ | 454/500 [00:23<00:02, 20.36it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  91%|█████████▏| 457/500 [00:23<00:02, 20.26it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  92%|█████████▏| 460/500 [00:23<00:01, 20.95it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (20):  93%|█████████▎| 463/500 [00:23<00:01, 21.03it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  93%|█████████▎| 466/500 [00:23<00:01, 20.73it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  94%|█████████▍| 469/500 [00:23<00:01, 20.89it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  94%|█████████▍| 472/500 [00:23<00:01, 20.18it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  95%|█████████▌| 475/500 [00:24<00:01, 20.47it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  96%|█████████▌| 478/500 [00:24<00:01, 20.35it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  96%|█████████▌| 481/500 [00:24<00:00, 20.92it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  97%|█████████▋| 484/500 [00:24<00:00, 21.30it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  97%|█████████▋| 487/500 [00:24<00:00, 21.70it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  98%|█████████▊| 490/500 [00:24<00:00, 21.73it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  99%|█████████▊| 493/500 [00:24<00:00, 21.28it/s]\u001b[A\n",
      "Searching with note10 constraint (20):  99%|█████████▉| 496/500 [00:25<00:00, 21.33it/s]\u001b[A\n",
      "Searching with note10 constraint (20): 100%|██████████| 500/500 [00:25<00:00, 19.82it/s]\u001b[A\n",
      "\n",
      "Searching with note10 constraint (25):   0%|          | 0/500 [00:00<?, ?it/s]\u001b[A\n",
      "Searching with note10 constraint (25):   0%|          | 2/500 [00:00<00:25, 19.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 20.00 msIt achieves 80.56% predicted accuracy with 19.95 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K3), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "((O96, E3.0, K7), None)\n",
      "((O96, E4.0, K5), Identity)\n",
      "((O96, E4.0, K3), Identity)\n",
      "(SE(O136, E4.0, K5), None)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E4.0, K7), Identity)\n",
      "(SE(O192, E3.0, K3), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  25.56589913368225\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Searching with note10 constraint (25):   1%|          | 4/500 [00:00<00:25, 19.10it/s]\u001b[A\n",
      "Searching with note10 constraint (25):   1%|          | 6/500 [00:00<00:25, 19.36it/s]\u001b[A\n",
      "Searching with note10 constraint (25):   2%|▏         | 9/500 [00:00<00:23, 20.63it/s]\u001b[A\n",
      "Searching with note10 constraint (25):   2%|▏         | 11/500 [00:00<00:24, 19.68it/s]\u001b[A\n",
      "Searching with note10 constraint (25):   3%|▎         | 13/500 [00:00<00:26, 18.72it/s]\u001b[A\n",
      "Searching with note10 constraint (25):   3%|▎         | 15/500 [00:00<00:25, 18.65it/s]\u001b[A\n",
      "Searching with note10 constraint (25):   3%|▎         | 17/500 [00:00<00:25, 18.97it/s]\u001b[A\n",
      "Searching with note10 constraint (25):   4%|▍         | 20/500 [00:01<00:24, 19.98it/s]\u001b[A\n",
      "Searching with note10 constraint (25):   5%|▍         | 23/500 [00:01<00:22, 21.05it/s]\u001b[A\n",
      "Searching with note10 constraint (25):   5%|▌         | 26/500 [00:01<00:21, 21.78it/s]\u001b[A\n",
      "Searching with note10 constraint (25):   6%|▌         | 29/500 [00:01<00:20, 22.61it/s]\u001b[A\n",
      "Searching with note10 constraint (25):   6%|▋         | 32/500 [00:01<00:20, 23.30it/s]\u001b[A\n",
      "Searching with note10 constraint (25):   7%|▋         | 35/500 [00:01<00:19, 23.95it/s]\u001b[A\n",
      "Searching with note10 constraint (25):   8%|▊         | 38/500 [00:01<00:19, 23.98it/s]\u001b[A\n",
      "Searching with note10 constraint (25):   8%|▊         | 41/500 [00:01<00:19, 24.06it/s]\u001b[A\n",
      "Searching with note10 constraint (25):   9%|▉         | 44/500 [00:01<00:18, 24.34it/s]\u001b[A\n",
      "Searching with note10 constraint (25):   9%|▉         | 47/500 [00:02<00:18, 23.91it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  10%|█         | 50/500 [00:02<00:19, 23.57it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  11%|█         | 53/500 [00:02<00:19, 22.91it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  11%|█         | 56/500 [00:02<00:20, 22.07it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  12%|█▏        | 59/500 [00:02<00:19, 22.44it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  12%|█▏        | 62/500 [00:02<00:19, 22.74it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  13%|█▎        | 65/500 [00:02<00:18, 23.00it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  14%|█▎        | 68/500 [00:03<00:18, 23.26it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  14%|█▍        | 71/500 [00:03<00:18, 22.96it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  15%|█▍        | 74/500 [00:03<00:18, 22.93it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  15%|█▌        | 77/500 [00:03<00:18, 22.90it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  16%|█▌        | 80/500 [00:03<00:18, 23.18it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  17%|█▋        | 83/500 [00:03<00:17, 23.38it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  17%|█▋        | 86/500 [00:03<00:17, 23.34it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  18%|█▊        | 89/500 [00:03<00:17, 23.70it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  18%|█▊        | 92/500 [00:04<00:17, 23.40it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  19%|█▉        | 95/500 [00:04<00:17, 23.57it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  20%|█▉        | 98/500 [00:04<00:17, 23.26it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  20%|██        | 101/500 [00:04<00:17, 23.21it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  21%|██        | 104/500 [00:04<00:17, 23.16it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  21%|██▏       | 107/500 [00:04<00:16, 23.23it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  22%|██▏       | 110/500 [00:04<00:17, 22.83it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  23%|██▎       | 113/500 [00:04<00:16, 22.77it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  23%|██▎       | 116/500 [00:05<00:16, 22.97it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  24%|██▍       | 119/500 [00:05<00:16, 23.15it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  24%|██▍       | 122/500 [00:05<00:16, 22.79it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  25%|██▌       | 125/500 [00:05<00:16, 22.86it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  26%|██▌       | 128/500 [00:05<00:16, 23.18it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  26%|██▌       | 131/500 [00:05<00:16, 22.94it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  27%|██▋       | 134/500 [00:05<00:16, 22.63it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  27%|██▋       | 137/500 [00:06<00:15, 22.92it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  28%|██▊       | 140/500 [00:06<00:15, 23.16it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  29%|██▊       | 143/500 [00:06<00:15, 23.29it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  29%|██▉       | 146/500 [00:06<00:15, 23.10it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  30%|██▉       | 149/500 [00:06<00:15, 22.74it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  30%|███       | 152/500 [00:06<00:14, 23.24it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  31%|███       | 155/500 [00:06<00:14, 23.13it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  32%|███▏      | 158/500 [00:06<00:14, 23.22it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  32%|███▏      | 161/500 [00:07<00:14, 22.80it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  33%|███▎      | 164/500 [00:07<00:15, 22.10it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  33%|███▎      | 167/500 [00:07<00:14, 22.26it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  34%|███▍      | 170/500 [00:07<00:15, 21.82it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  35%|███▍      | 173/500 [00:07<00:14, 22.23it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  35%|███▌      | 176/500 [00:07<00:14, 22.45it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  36%|███▌      | 179/500 [00:07<00:14, 22.54it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  36%|███▋      | 182/500 [00:08<00:14, 21.95it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  37%|███▋      | 185/500 [00:08<00:14, 21.48it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  38%|███▊      | 188/500 [00:08<00:14, 20.92it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  38%|███▊      | 191/500 [00:08<00:14, 21.33it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  39%|███▉      | 194/500 [00:08<00:13, 21.94it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  39%|███▉      | 197/500 [00:08<00:13, 22.07it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  40%|████      | 200/500 [00:08<00:13, 21.69it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  41%|████      | 203/500 [00:09<00:13, 21.71it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  41%|████      | 206/500 [00:09<00:13, 21.77it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  42%|████▏     | 209/500 [00:09<00:13, 22.28it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  42%|████▏     | 212/500 [00:09<00:13, 21.96it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  43%|████▎     | 215/500 [00:09<00:13, 21.79it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  44%|████▎     | 218/500 [00:09<00:12, 21.95it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  44%|████▍     | 221/500 [00:09<00:12, 21.79it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  45%|████▍     | 224/500 [00:09<00:12, 21.84it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  45%|████▌     | 227/500 [00:10<00:12, 21.95it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  46%|████▌     | 230/500 [00:10<00:12, 22.42it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  47%|████▋     | 233/500 [00:10<00:11, 22.34it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  47%|████▋     | 236/500 [00:10<00:11, 22.54it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  48%|████▊     | 239/500 [00:10<00:11, 22.37it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  48%|████▊     | 242/500 [00:10<00:11, 22.66it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  49%|████▉     | 245/500 [00:10<00:11, 22.25it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  50%|████▉     | 248/500 [00:11<00:11, 21.97it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  50%|█████     | 251/500 [00:11<00:11, 22.22it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  51%|█████     | 254/500 [00:11<00:11, 21.17it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  51%|█████▏    | 257/500 [00:11<00:12, 20.15it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  52%|█████▏    | 260/500 [00:11<00:11, 20.74it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  53%|█████▎    | 263/500 [00:11<00:11, 20.93it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  53%|█████▎    | 266/500 [00:11<00:11, 21.20it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  54%|█████▍    | 269/500 [00:12<00:10, 21.40it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  54%|█████▍    | 272/500 [00:12<00:10, 21.88it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  55%|█████▌    | 275/500 [00:12<00:10, 22.20it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  56%|█████▌    | 278/500 [00:12<00:09, 22.68it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  56%|█████▌    | 281/500 [00:12<00:09, 22.80it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  57%|█████▋    | 284/500 [00:12<00:09, 22.56it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  57%|█████▋    | 287/500 [00:12<00:09, 22.57it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  58%|█████▊    | 290/500 [00:12<00:09, 22.89it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  59%|█████▊    | 293/500 [00:13<00:09, 22.67it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  59%|█████▉    | 296/500 [00:13<00:08, 22.68it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  60%|█████▉    | 299/500 [00:13<00:08, 23.01it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  60%|██████    | 302/500 [00:13<00:08, 22.85it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  61%|██████    | 305/500 [00:13<00:08, 23.48it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  62%|██████▏   | 308/500 [00:13<00:08, 23.47it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  62%|██████▏   | 311/500 [00:13<00:08, 23.32it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  63%|██████▎   | 314/500 [00:13<00:07, 23.54it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  63%|██████▎   | 317/500 [00:14<00:07, 23.36it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  64%|██████▍   | 320/500 [00:14<00:07, 23.44it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  65%|██████▍   | 323/500 [00:14<00:07, 22.89it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  65%|██████▌   | 326/500 [00:14<00:07, 22.77it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  66%|██████▌   | 329/500 [00:14<00:07, 22.75it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  66%|██████▋   | 332/500 [00:14<00:07, 23.17it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  67%|██████▋   | 335/500 [00:14<00:07, 23.07it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  68%|██████▊   | 338/500 [00:15<00:07, 23.03it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  68%|██████▊   | 341/500 [00:15<00:06, 23.45it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  69%|██████▉   | 344/500 [00:15<00:06, 22.78it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  69%|██████▉   | 347/500 [00:15<00:06, 22.51it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  70%|███████   | 350/500 [00:15<00:06, 22.65it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  71%|███████   | 353/500 [00:15<00:06, 22.46it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  71%|███████   | 356/500 [00:15<00:06, 22.60it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  72%|███████▏  | 359/500 [00:15<00:06, 22.35it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  72%|███████▏  | 362/500 [00:16<00:06, 22.30it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  73%|███████▎  | 365/500 [00:16<00:05, 22.58it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  74%|███████▎  | 368/500 [00:16<00:06, 21.63it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  74%|███████▍  | 371/500 [00:16<00:06, 21.10it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  75%|███████▍  | 374/500 [00:16<00:06, 20.86it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  75%|███████▌  | 377/500 [00:16<00:05, 21.00it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  76%|███████▌  | 380/500 [00:16<00:05, 21.23it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  77%|███████▋  | 383/500 [00:17<00:05, 21.59it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  77%|███████▋  | 386/500 [00:17<00:05, 22.12it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  78%|███████▊  | 389/500 [00:17<00:04, 22.41it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  78%|███████▊  | 392/500 [00:17<00:04, 22.78it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  79%|███████▉  | 395/500 [00:17<00:04, 22.97it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  80%|███████▉  | 398/500 [00:17<00:04, 23.39it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  80%|████████  | 401/500 [00:17<00:04, 23.16it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  81%|████████  | 404/500 [00:17<00:04, 23.00it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  81%|████████▏ | 407/500 [00:18<00:04, 22.95it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  82%|████████▏ | 410/500 [00:18<00:03, 22.66it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  83%|████████▎ | 413/500 [00:18<00:03, 22.63it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  83%|████████▎ | 416/500 [00:18<00:03, 22.79it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  84%|████████▍ | 419/500 [00:18<00:03, 22.76it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  84%|████████▍ | 422/500 [00:18<00:03, 22.88it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  85%|████████▌ | 425/500 [00:18<00:03, 22.96it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  86%|████████▌ | 428/500 [00:19<00:03, 22.95it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  86%|████████▌ | 431/500 [00:19<00:03, 22.99it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  87%|████████▋ | 434/500 [00:19<00:02, 23.11it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  87%|████████▋ | 437/500 [00:19<00:02, 22.97it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  88%|████████▊ | 440/500 [00:19<00:02, 22.90it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  89%|████████▊ | 443/500 [00:19<00:02, 23.13it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  89%|████████▉ | 446/500 [00:19<00:02, 23.36it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  90%|████████▉ | 449/500 [00:19<00:02, 23.29it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  90%|█████████ | 452/500 [00:20<00:02, 23.37it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  91%|█████████ | 455/500 [00:20<00:01, 23.03it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  92%|█████████▏| 458/500 [00:20<00:01, 23.50it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  92%|█████████▏| 461/500 [00:20<00:01, 23.32it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  93%|█████████▎| 464/500 [00:20<00:01, 23.43it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  93%|█████████▎| 467/500 [00:20<00:01, 22.73it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  94%|█████████▍| 470/500 [00:20<00:01, 22.87it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  95%|█████████▍| 473/500 [00:21<00:01, 22.57it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  95%|█████████▌| 476/500 [00:21<00:01, 22.30it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  96%|█████████▌| 479/500 [00:21<00:00, 22.32it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  96%|█████████▋| 482/500 [00:21<00:00, 22.30it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  97%|█████████▋| 485/500 [00:21<00:00, 22.39it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  98%|█████████▊| 488/500 [00:21<00:00, 21.93it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  98%|█████████▊| 491/500 [00:21<00:00, 21.99it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  99%|█████████▉| 494/500 [00:21<00:00, 21.70it/s]\u001b[A\n",
      "Searching with note10 constraint (25):  99%|█████████▉| 497/500 [00:22<00:00, 22.28it/s]\u001b[A\n",
      "Searching with note10 constraint (25): 100%|██████████| 500/500 [00:22<00:00, 22.50it/s]\u001b[A\n",
      "\n",
      "Searching with note10 constraint (30):   0%|          | 0/500 [00:00<?, ?it/s]\u001b[A\n",
      "Searching with note10 constraint (30):   1%|          | 3/500 [00:00<00:20, 24.61it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 25.00 msIt achieves 81.80% predicted accuracy with 24.92 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E4.0, K5), Identity)\n",
      "((O96, E4.0, K3), Identity)\n",
      "((O96, E3.0, K7), Identity)\n",
      "(SE(O136, E4.0, K7), None)\n",
      "(SE(O136, E4.0, K3), Identity)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  22.32000708580017\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Searching with note10 constraint (30):   1%|          | 6/500 [00:00<00:20, 24.22it/s]\u001b[A\n",
      "Searching with note10 constraint (30):   2%|▏         | 9/500 [00:00<00:20, 24.23it/s]\u001b[A\n",
      "Searching with note10 constraint (30):   2%|▏         | 12/500 [00:00<00:20, 23.69it/s]\u001b[A\n",
      "Searching with note10 constraint (30):   3%|▎         | 15/500 [00:00<00:20, 23.45it/s]\u001b[A\n",
      "Searching with note10 constraint (30):   4%|▎         | 18/500 [00:00<00:20, 23.06it/s]\u001b[A\n",
      "Searching with note10 constraint (30):   4%|▍         | 21/500 [00:00<00:20, 23.53it/s]\u001b[A\n",
      "Searching with note10 constraint (30):   5%|▍         | 24/500 [00:01<00:19, 24.27it/s]\u001b[A\n",
      "Searching with note10 constraint (30):   5%|▌         | 27/500 [00:01<00:19, 24.08it/s]\u001b[A\n",
      "Searching with note10 constraint (30):   6%|▌         | 30/500 [00:01<00:19, 24.00it/s]\u001b[A\n",
      "Searching with note10 constraint (30):   7%|▋         | 33/500 [00:01<00:19, 23.91it/s]\u001b[A\n",
      "Searching with note10 constraint (30):   7%|▋         | 36/500 [00:01<00:19, 24.22it/s]\u001b[A\n",
      "Searching with note10 constraint (30):   8%|▊         | 39/500 [00:01<00:18, 24.42it/s]\u001b[A\n",
      "Searching with note10 constraint (30):   8%|▊         | 42/500 [00:01<00:18, 24.87it/s]\u001b[A\n",
      "Searching with note10 constraint (30):   9%|▉         | 45/500 [00:01<00:18, 25.10it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  10%|▉         | 48/500 [00:01<00:17, 25.48it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  10%|█         | 51/500 [00:02<00:17, 25.47it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  11%|█         | 54/500 [00:02<00:17, 25.56it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  11%|█▏        | 57/500 [00:02<00:17, 25.75it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  12%|█▏        | 60/500 [00:02<00:16, 26.13it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  13%|█▎        | 63/500 [00:02<00:16, 26.49it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  13%|█▎        | 66/500 [00:02<00:16, 25.67it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  14%|█▍        | 69/500 [00:02<00:16, 25.64it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  14%|█▍        | 72/500 [00:02<00:16, 25.77it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  15%|█▌        | 75/500 [00:03<00:16, 26.31it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  16%|█▌        | 78/500 [00:03<00:16, 25.87it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  16%|█▌        | 81/500 [00:03<00:16, 25.71it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  17%|█▋        | 84/500 [00:03<00:16, 25.39it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  17%|█▋        | 87/500 [00:03<00:16, 25.35it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  18%|█▊        | 90/500 [00:03<00:16, 25.16it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  19%|█▊        | 93/500 [00:03<00:16, 24.32it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  19%|█▉        | 96/500 [00:03<00:16, 24.14it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  20%|█▉        | 99/500 [00:03<00:16, 24.08it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  20%|██        | 102/500 [00:04<00:16, 24.10it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  21%|██        | 105/500 [00:04<00:15, 24.79it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  22%|██▏       | 108/500 [00:04<00:15, 24.95it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  22%|██▏       | 111/500 [00:04<00:15, 24.78it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  23%|██▎       | 114/500 [00:04<00:15, 25.07it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  23%|██▎       | 117/500 [00:04<00:15, 25.09it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  24%|██▍       | 120/500 [00:04<00:15, 25.23it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  25%|██▍       | 123/500 [00:04<00:14, 25.38it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  25%|██▌       | 126/500 [00:05<00:15, 24.80it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  26%|██▌       | 129/500 [00:05<00:15, 23.86it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  26%|██▋       | 132/500 [00:05<00:15, 23.51it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  27%|██▋       | 135/500 [00:05<00:15, 23.85it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  28%|██▊       | 138/500 [00:05<00:15, 24.03it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  28%|██▊       | 141/500 [00:05<00:14, 24.19it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  29%|██▉       | 144/500 [00:05<00:14, 24.74it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  29%|██▉       | 147/500 [00:05<00:14, 24.85it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  30%|███       | 150/500 [00:06<00:14, 24.87it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  31%|███       | 153/500 [00:06<00:13, 24.83it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  31%|███       | 156/500 [00:06<00:13, 24.89it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  32%|███▏      | 159/500 [00:06<00:13, 24.79it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  32%|███▏      | 162/500 [00:06<00:13, 24.91it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  33%|███▎      | 165/500 [00:06<00:13, 24.81it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  34%|███▎      | 168/500 [00:06<00:13, 24.30it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  34%|███▍      | 171/500 [00:06<00:13, 24.52it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  35%|███▍      | 174/500 [00:07<00:12, 25.19it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  35%|███▌      | 177/500 [00:07<00:12, 25.05it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  36%|███▌      | 180/500 [00:07<00:12, 24.73it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  37%|███▋      | 183/500 [00:07<00:12, 24.73it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  37%|███▋      | 186/500 [00:07<00:12, 25.01it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  38%|███▊      | 189/500 [00:07<00:12, 24.91it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  38%|███▊      | 192/500 [00:07<00:12, 24.56it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  39%|███▉      | 195/500 [00:07<00:12, 24.50it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  40%|███▉      | 198/500 [00:08<00:12, 24.53it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  40%|████      | 201/500 [00:08<00:12, 24.52it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  41%|████      | 204/500 [00:08<00:12, 24.48it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  41%|████▏     | 207/500 [00:08<00:12, 23.44it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  42%|████▏     | 210/500 [00:08<00:12, 23.70it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  43%|████▎     | 213/500 [00:08<00:11, 24.52it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  43%|████▎     | 216/500 [00:08<00:11, 24.12it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  44%|████▍     | 219/500 [00:08<00:11, 23.94it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  44%|████▍     | 222/500 [00:09<00:11, 23.22it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  45%|████▌     | 225/500 [00:09<00:11, 23.89it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  46%|████▌     | 228/500 [00:09<00:11, 23.41it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  46%|████▌     | 231/500 [00:09<00:11, 22.76it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  47%|████▋     | 234/500 [00:09<00:11, 22.31it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  47%|████▋     | 237/500 [00:09<00:11, 22.50it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  48%|████▊     | 240/500 [00:09<00:11, 23.55it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  49%|████▊     | 243/500 [00:09<00:10, 23.87it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  49%|████▉     | 246/500 [00:10<00:10, 24.09it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  50%|████▉     | 249/500 [00:10<00:10, 24.05it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  50%|█████     | 252/500 [00:10<00:10, 23.95it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  51%|█████     | 255/500 [00:10<00:09, 24.77it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  52%|█████▏    | 258/500 [00:10<00:09, 24.81it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  52%|█████▏    | 261/500 [00:10<00:09, 24.71it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  53%|█████▎    | 264/500 [00:10<00:09, 24.82it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  53%|█████▎    | 267/500 [00:10<00:09, 24.91it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  54%|█████▍    | 270/500 [00:11<00:09, 25.27it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  55%|█████▍    | 273/500 [00:11<00:08, 25.38it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  55%|█████▌    | 276/500 [00:11<00:09, 24.63it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  56%|█████▌    | 279/500 [00:11<00:08, 24.83it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  56%|█████▋    | 282/500 [00:11<00:09, 24.18it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  57%|█████▋    | 285/500 [00:11<00:09, 22.53it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  58%|█████▊    | 288/500 [00:11<00:09, 22.81it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  58%|█████▊    | 291/500 [00:11<00:08, 23.36it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  59%|█████▉    | 294/500 [00:12<00:08, 24.17it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  59%|█████▉    | 297/500 [00:12<00:08, 24.45it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  60%|██████    | 300/500 [00:12<00:08, 24.92it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  61%|██████    | 303/500 [00:12<00:07, 24.81it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  61%|██████    | 306/500 [00:12<00:07, 24.31it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  62%|██████▏   | 309/500 [00:12<00:07, 24.09it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  62%|██████▏   | 312/500 [00:12<00:07, 23.90it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  63%|██████▎   | 315/500 [00:12<00:07, 24.42it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  64%|██████▎   | 318/500 [00:12<00:07, 25.13it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  64%|██████▍   | 321/500 [00:13<00:06, 26.00it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  65%|██████▍   | 324/500 [00:13<00:06, 25.62it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  65%|██████▌   | 327/500 [00:13<00:06, 25.20it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  66%|██████▌   | 330/500 [00:13<00:06, 24.75it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  67%|██████▋   | 333/500 [00:13<00:06, 24.66it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  67%|██████▋   | 336/500 [00:13<00:06, 24.15it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  68%|██████▊   | 339/500 [00:13<00:06, 23.58it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  68%|██████▊   | 342/500 [00:13<00:06, 23.77it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  69%|██████▉   | 345/500 [00:14<00:06, 23.20it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  70%|██████▉   | 348/500 [00:14<00:06, 23.07it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  70%|███████   | 351/500 [00:14<00:06, 22.75it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  71%|███████   | 354/500 [00:14<00:06, 23.05it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  71%|███████▏  | 357/500 [00:14<00:06, 22.79it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  72%|███████▏  | 360/500 [00:14<00:06, 22.96it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  73%|███████▎  | 363/500 [00:14<00:06, 22.79it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  73%|███████▎  | 366/500 [00:15<00:05, 22.99it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  74%|███████▍  | 369/500 [00:15<00:05, 23.12it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  74%|███████▍  | 372/500 [00:15<00:05, 23.00it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  75%|███████▌  | 375/500 [00:15<00:05, 23.04it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  76%|███████▌  | 378/500 [00:15<00:05, 23.03it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  76%|███████▌  | 381/500 [00:15<00:05, 23.26it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  77%|███████▋  | 384/500 [00:15<00:04, 23.40it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  77%|███████▋  | 387/500 [00:15<00:04, 23.37it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  78%|███████▊  | 390/500 [00:16<00:04, 23.71it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  79%|███████▊  | 393/500 [00:16<00:04, 24.40it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  79%|███████▉  | 396/500 [00:16<00:04, 24.71it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  80%|███████▉  | 399/500 [00:16<00:03, 25.57it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  80%|████████  | 402/500 [00:16<00:03, 24.90it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  81%|████████  | 405/500 [00:16<00:03, 24.94it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  82%|████████▏ | 408/500 [00:16<00:03, 24.69it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  82%|████████▏ | 411/500 [00:16<00:03, 24.33it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  83%|████████▎ | 414/500 [00:17<00:03, 24.78it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  83%|████████▎ | 417/500 [00:17<00:03, 24.67it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  84%|████████▍ | 420/500 [00:17<00:03, 24.90it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  85%|████████▍ | 423/500 [00:17<00:03, 25.34it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  85%|████████▌ | 426/500 [00:17<00:02, 25.87it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  86%|████████▌ | 429/500 [00:17<00:02, 26.25it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  86%|████████▋ | 432/500 [00:17<00:02, 26.33it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  87%|████████▋ | 435/500 [00:17<00:02, 26.56it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  88%|████████▊ | 438/500 [00:17<00:02, 26.71it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  88%|████████▊ | 441/500 [00:18<00:02, 26.36it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  89%|████████▉ | 444/500 [00:18<00:02, 26.70it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  89%|████████▉ | 447/500 [00:18<00:01, 26.62it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  90%|█████████ | 450/500 [00:18<00:01, 26.73it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  91%|█████████ | 453/500 [00:18<00:01, 26.08it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  91%|█████████ | 456/500 [00:18<00:01, 25.02it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  92%|█████████▏| 459/500 [00:18<00:01, 24.21it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  92%|█████████▏| 462/500 [00:18<00:01, 23.88it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  93%|█████████▎| 465/500 [00:19<00:01, 23.98it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  94%|█████████▎| 468/500 [00:19<00:01, 23.94it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  94%|█████████▍| 471/500 [00:19<00:01, 23.08it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  95%|█████████▍| 474/500 [00:19<00:01, 23.62it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  95%|█████████▌| 477/500 [00:19<00:00, 24.31it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  96%|█████████▌| 480/500 [00:19<00:00, 24.15it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  97%|█████████▋| 483/500 [00:19<00:00, 24.20it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  97%|█████████▋| 486/500 [00:19<00:00, 24.68it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  98%|█████████▊| 489/500 [00:19<00:00, 24.99it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  98%|█████████▊| 492/500 [00:20<00:00, 25.23it/s]\u001b[A\n",
      "Searching with note10 constraint (30):  99%|█████████▉| 495/500 [00:20<00:00, 25.22it/s]\u001b[A\n",
      "Searching with note10 constraint (30): 100%|██████████| 500/500 [00:20<00:00, 24.49it/s]\u001b[A\n",
      "\n",
      "Searching with note10 constraint (35):   0%|          | 0/500 [00:00<?, ?it/s]\u001b[A\n",
      "Searching with note10 constraint (35):   1%|          | 3/500 [00:00<00:20, 24.47it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 30.00 msIt achieves 82.78% predicted accuracy with 29.38 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E4.0, K3), Identity)\n",
      "(SE(O48, E6.0, K5), None)\n",
      "(SE(O48, E4.0, K7), Identity)\n",
      "(SE(O48, E3.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E3.0, K3), Identity)\n",
      "((O96, E4.0, K3), Identity)\n",
      "((O96, E3.0, K3), Identity)\n",
      "(SE(O136, E6.0, K3), None)\n",
      "(SE(O136, E4.0, K7), Identity)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O136, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E3.0, K3), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  20.48673677444458\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Searching with note10 constraint (35):   1%|          | 6/500 [00:00<00:20, 23.94it/s]\u001b[A\n",
      "Searching with note10 constraint (35):   2%|▏         | 9/500 [00:00<00:20, 24.24it/s]\u001b[A\n",
      "Searching with note10 constraint (35):   2%|▏         | 12/500 [00:00<00:19, 24.92it/s]\u001b[A\n",
      "Searching with note10 constraint (35):   3%|▎         | 15/500 [00:00<00:19, 25.39it/s]\u001b[A\n",
      "Searching with note10 constraint (35):   4%|▎         | 18/500 [00:00<00:18, 25.69it/s]\u001b[A\n",
      "Searching with note10 constraint (35):   4%|▍         | 21/500 [00:00<00:19, 24.77it/s]\u001b[A\n",
      "Searching with note10 constraint (35):   5%|▍         | 24/500 [00:00<00:19, 23.93it/s]\u001b[A\n",
      "Searching with note10 constraint (35):   5%|▌         | 27/500 [00:01<00:19, 24.62it/s]\u001b[A\n",
      "Searching with note10 constraint (35):   6%|▌         | 30/500 [00:01<00:19, 24.50it/s]\u001b[A\n",
      "Searching with note10 constraint (35):   7%|▋         | 33/500 [00:01<00:18, 24.60it/s]\u001b[A\n",
      "Searching with note10 constraint (35):   7%|▋         | 36/500 [00:01<00:18, 24.51it/s]\u001b[A\n",
      "Searching with note10 constraint (35):   8%|▊         | 39/500 [00:01<00:19, 24.14it/s]\u001b[A\n",
      "Searching with note10 constraint (35):   8%|▊         | 42/500 [00:01<00:19, 23.72it/s]\u001b[A\n",
      "Searching with note10 constraint (35):   9%|▉         | 45/500 [00:01<00:19, 23.65it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  10%|▉         | 48/500 [00:01<00:19, 23.73it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  10%|█         | 51/500 [00:02<00:18, 24.40it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  11%|█         | 54/500 [00:02<00:18, 23.94it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  11%|█▏        | 57/500 [00:02<00:18, 24.11it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  12%|█▏        | 60/500 [00:02<00:18, 24.12it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  13%|█▎        | 63/500 [00:02<00:18, 24.14it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  13%|█▎        | 66/500 [00:02<00:17, 24.45it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  14%|█▍        | 69/500 [00:02<00:17, 24.65it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  14%|█▍        | 72/500 [00:02<00:17, 25.00it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  15%|█▌        | 75/500 [00:03<00:17, 24.35it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  16%|█▌        | 78/500 [00:03<00:17, 24.53it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  16%|█▌        | 81/500 [00:03<00:17, 24.50it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  17%|█▋        | 84/500 [00:03<00:17, 24.25it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  17%|█▋        | 87/500 [00:03<00:17, 23.92it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  18%|█▊        | 90/500 [00:03<00:17, 23.32it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  19%|█▊        | 93/500 [00:03<00:17, 23.46it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  19%|█▉        | 96/500 [00:03<00:17, 23.15it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  20%|█▉        | 99/500 [00:04<00:16, 23.81it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  20%|██        | 102/500 [00:04<00:16, 23.56it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  21%|██        | 105/500 [00:04<00:17, 23.21it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  22%|██▏       | 108/500 [00:04<00:16, 23.63it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  22%|██▏       | 111/500 [00:04<00:16, 23.73it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  23%|██▎       | 114/500 [00:04<00:16, 24.09it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  23%|██▎       | 117/500 [00:04<00:15, 24.26it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  24%|██▍       | 120/500 [00:04<00:15, 24.89it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  25%|██▍       | 123/500 [00:05<00:15, 25.11it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  25%|██▌       | 126/500 [00:05<00:15, 24.67it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  26%|██▌       | 129/500 [00:05<00:15, 24.35it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  26%|██▋       | 132/500 [00:05<00:15, 24.34it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  27%|██▋       | 135/500 [00:05<00:14, 24.52it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  28%|██▊       | 138/500 [00:05<00:14, 24.64it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  28%|██▊       | 141/500 [00:05<00:14, 24.55it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  29%|██▉       | 144/500 [00:05<00:14, 24.73it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  29%|██▉       | 147/500 [00:06<00:13, 25.54it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  30%|███       | 150/500 [00:06<00:13, 25.63it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  31%|███       | 153/500 [00:06<00:13, 25.19it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  31%|███       | 156/500 [00:06<00:13, 24.89it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  32%|███▏      | 159/500 [00:06<00:13, 24.43it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  32%|███▏      | 162/500 [00:06<00:14, 24.09it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  33%|███▎      | 165/500 [00:06<00:14, 23.90it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  34%|███▎      | 168/500 [00:06<00:13, 23.98it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  34%|███▍      | 171/500 [00:07<00:13, 24.25it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  35%|███▍      | 174/500 [00:07<00:13, 24.44it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  35%|███▌      | 177/500 [00:07<00:13, 24.73it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  36%|███▌      | 180/500 [00:07<00:12, 24.65it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  37%|███▋      | 183/500 [00:07<00:12, 24.96it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  37%|███▋      | 186/500 [00:07<00:12, 25.25it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  38%|███▊      | 189/500 [00:07<00:12, 25.33it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  38%|███▊      | 192/500 [00:07<00:12, 24.87it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  39%|███▉      | 195/500 [00:07<00:12, 24.45it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  40%|███▉      | 198/500 [00:08<00:12, 24.39it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  40%|████      | 201/500 [00:08<00:12, 24.13it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  41%|████      | 204/500 [00:08<00:11, 24.82it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  41%|████▏     | 207/500 [00:08<00:11, 25.07it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  42%|████▏     | 210/500 [00:08<00:11, 24.79it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  43%|████▎     | 213/500 [00:08<00:11, 24.02it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  43%|████▎     | 216/500 [00:08<00:12, 23.61it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  44%|████▍     | 219/500 [00:08<00:11, 23.42it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  44%|████▍     | 222/500 [00:09<00:11, 23.23it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  45%|████▌     | 225/500 [00:09<00:12, 21.50it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  46%|████▌     | 228/500 [00:09<00:12, 21.66it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  46%|████▌     | 231/500 [00:09<00:13, 19.92it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  47%|████▋     | 234/500 [00:09<00:13, 20.39it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  47%|████▋     | 237/500 [00:09<00:13, 19.08it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  48%|████▊     | 239/500 [00:10<00:14, 18.03it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  48%|████▊     | 241/500 [00:10<00:14, 18.46it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  49%|████▊     | 243/500 [00:10<00:13, 18.71it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  49%|████▉     | 246/500 [00:10<00:12, 19.66it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  50%|████▉     | 249/500 [00:10<00:11, 21.04it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  50%|█████     | 252/500 [00:10<00:11, 22.05it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  51%|█████     | 255/500 [00:10<00:10, 22.42it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  52%|█████▏    | 258/500 [00:10<00:10, 22.72it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  52%|█████▏    | 261/500 [00:11<00:10, 23.49it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  53%|█████▎    | 264/500 [00:11<00:09, 24.07it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  53%|█████▎    | 267/500 [00:11<00:09, 24.27it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  54%|█████▍    | 270/500 [00:11<00:09, 24.15it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  55%|█████▍    | 273/500 [00:11<00:09, 24.67it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  55%|█████▌    | 276/500 [00:11<00:08, 24.97it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  56%|█████▌    | 279/500 [00:11<00:08, 24.86it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  56%|█████▋    | 282/500 [00:11<00:08, 24.77it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  57%|█████▋    | 285/500 [00:11<00:08, 24.77it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  58%|█████▊    | 288/500 [00:12<00:08, 25.31it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  58%|█████▊    | 291/500 [00:12<00:08, 25.57it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  59%|█████▉    | 294/500 [00:12<00:07, 26.10it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  59%|█████▉    | 297/500 [00:12<00:07, 26.05it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  60%|██████    | 300/500 [00:12<00:07, 25.92it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  61%|██████    | 303/500 [00:12<00:07, 25.58it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  61%|██████    | 306/500 [00:12<00:07, 26.13it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  62%|██████▏   | 309/500 [00:12<00:07, 26.40it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  62%|██████▏   | 312/500 [00:12<00:07, 26.19it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  63%|██████▎   | 315/500 [00:13<00:07, 25.91it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  64%|██████▎   | 318/500 [00:13<00:07, 25.73it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  64%|██████▍   | 321/500 [00:13<00:07, 24.49it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  65%|██████▍   | 324/500 [00:13<00:07, 23.72it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  65%|██████▌   | 327/500 [00:13<00:07, 23.55it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  66%|██████▌   | 330/500 [00:13<00:07, 22.07it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  67%|██████▋   | 333/500 [00:13<00:07, 21.99it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  67%|██████▋   | 336/500 [00:14<00:07, 22.79it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  68%|██████▊   | 339/500 [00:14<00:07, 22.97it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  68%|██████▊   | 342/500 [00:14<00:06, 23.03it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  69%|██████▉   | 345/500 [00:14<00:06, 22.31it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  70%|██████▉   | 348/500 [00:14<00:06, 21.84it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  70%|███████   | 351/500 [00:14<00:06, 22.48it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  71%|███████   | 354/500 [00:14<00:06, 22.47it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  71%|███████▏  | 357/500 [00:14<00:06, 22.44it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  72%|███████▏  | 360/500 [00:15<00:06, 22.85it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  73%|███████▎  | 363/500 [00:15<00:06, 22.69it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  73%|███████▎  | 366/500 [00:15<00:05, 22.52it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  74%|███████▍  | 369/500 [00:15<00:05, 22.68it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  74%|███████▍  | 372/500 [00:15<00:05, 21.53it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  75%|███████▌  | 375/500 [00:15<00:05, 21.56it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  76%|███████▌  | 378/500 [00:15<00:05, 20.97it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  76%|███████▌  | 381/500 [00:16<00:05, 21.44it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  77%|███████▋  | 384/500 [00:16<00:05, 21.94it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  77%|███████▋  | 387/500 [00:16<00:05, 21.49it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  78%|███████▊  | 390/500 [00:16<00:04, 22.12it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  79%|███████▊  | 393/500 [00:16<00:04, 22.88it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  79%|███████▉  | 396/500 [00:16<00:04, 22.97it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  80%|███████▉  | 399/500 [00:16<00:04, 22.36it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  80%|████████  | 402/500 [00:17<00:04, 22.49it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  81%|████████  | 405/500 [00:17<00:04, 23.08it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  82%|████████▏ | 408/500 [00:17<00:03, 23.04it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  82%|████████▏ | 411/500 [00:17<00:04, 22.03it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  83%|████████▎ | 414/500 [00:17<00:03, 22.65it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  83%|████████▎ | 417/500 [00:17<00:03, 23.16it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  84%|████████▍ | 420/500 [00:17<00:03, 23.79it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  85%|████████▍ | 423/500 [00:17<00:03, 23.16it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  85%|████████▌ | 426/500 [00:18<00:03, 23.13it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  86%|████████▌ | 429/500 [00:18<00:03, 23.63it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  86%|████████▋ | 432/500 [00:18<00:02, 24.13it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  87%|████████▋ | 435/500 [00:18<00:02, 24.47it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  88%|████████▊ | 438/500 [00:18<00:02, 23.85it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  88%|████████▊ | 441/500 [00:18<00:02, 23.49it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  89%|████████▉ | 444/500 [00:18<00:02, 22.71it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  89%|████████▉ | 447/500 [00:18<00:02, 22.85it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  90%|█████████ | 450/500 [00:19<00:02, 22.73it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  91%|█████████ | 453/500 [00:19<00:02, 22.04it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  91%|█████████ | 456/500 [00:19<00:02, 21.61it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  92%|█████████▏| 459/500 [00:19<00:01, 22.17it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  92%|█████████▏| 462/500 [00:19<00:01, 21.63it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  93%|█████████▎| 465/500 [00:19<00:01, 21.55it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  94%|█████████▎| 468/500 [00:19<00:01, 22.43it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  94%|█████████▍| 471/500 [00:20<00:01, 23.18it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  95%|█████████▍| 474/500 [00:20<00:01, 23.32it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  95%|█████████▌| 477/500 [00:20<00:00, 23.61it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  96%|█████████▌| 480/500 [00:20<00:00, 24.02it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  97%|█████████▋| 483/500 [00:20<00:00, 24.38it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  97%|█████████▋| 486/500 [00:20<00:00, 24.13it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  98%|█████████▊| 489/500 [00:20<00:00, 24.10it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  98%|█████████▊| 492/500 [00:20<00:00, 24.51it/s]\u001b[A\n",
      "Searching with note10 constraint (35):  99%|█████████▉| 495/500 [00:21<00:00, 24.73it/s]\u001b[A\n",
      "Searching with note10 constraint (35): 100%|██████████| 500/500 [00:21<00:00, 23.58it/s]\u001b[A\n",
      "\n",
      "Searching with note10 constraint (40):   0%|          | 0/500 [00:00<?, ?it/s]\u001b[A\n",
      "Searching with note10 constraint (40):   1%|          | 3/500 [00:00<00:18, 26.27it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 35.00 msIt achieves 83.03% predicted accuracy with 34.63 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E4.0, K5), None)\n",
      "((O32, E6.0, K3), Identity)\n",
      "((O32, E3.0, K5), Identity)\n",
      "(SE(O48, E4.0, K5), None)\n",
      "(SE(O48, E4.0, K7), Identity)\n",
      "(SE(O48, E3.0, K5), Identity)\n",
      "((O96, E6.0, K3), None)\n",
      "((O96, E3.0, K5), Identity)\n",
      "((O96, E4.0, K3), Identity)\n",
      "((O96, E3.0, K5), Identity)\n",
      "(SE(O136, E6.0, K5), None)\n",
      "(SE(O136, E6.0, K3), Identity)\n",
      "(SE(O136, E6.0, K3), Identity)\n",
      "(SE(O136, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E3.0, K7), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  21.243536949157715\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Searching with note10 constraint (40):   1%|          | 6/500 [00:00<00:19, 25.11it/s]\u001b[A\n",
      "Searching with note10 constraint (40):   2%|▏         | 9/500 [00:00<00:20, 23.97it/s]\u001b[A\n",
      "Searching with note10 constraint (40):   2%|▏         | 12/500 [00:00<00:21, 23.12it/s]\u001b[A\n",
      "Searching with note10 constraint (40):   3%|▎         | 15/500 [00:00<00:20, 23.30it/s]\u001b[A\n",
      "Searching with note10 constraint (40):   4%|▎         | 18/500 [00:00<00:20, 23.25it/s]\u001b[A\n",
      "Searching with note10 constraint (40):   4%|▍         | 21/500 [00:00<00:19, 24.04it/s]\u001b[A\n",
      "Searching with note10 constraint (40):   5%|▍         | 24/500 [00:01<00:19, 24.60it/s]\u001b[A\n",
      "Searching with note10 constraint (40):   5%|▌         | 27/500 [00:01<00:18, 25.19it/s]\u001b[A\n",
      "Searching with note10 constraint (40):   6%|▌         | 30/500 [00:01<00:18, 24.90it/s]\u001b[A\n",
      "Searching with note10 constraint (40):   7%|▋         | 33/500 [00:01<00:19, 24.50it/s]\u001b[A\n",
      "Searching with note10 constraint (40):   7%|▋         | 36/500 [00:01<00:19, 24.02it/s]\u001b[A\n",
      "Searching with note10 constraint (40):   8%|▊         | 39/500 [00:01<00:19, 23.81it/s]\u001b[A\n",
      "Searching with note10 constraint (40):   8%|▊         | 42/500 [00:01<00:19, 23.77it/s]\u001b[A\n",
      "Searching with note10 constraint (40):   9%|▉         | 45/500 [00:01<00:18, 24.09it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  10%|▉         | 48/500 [00:02<00:18, 24.28it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  10%|█         | 51/500 [00:02<00:18, 24.12it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  11%|█         | 54/500 [00:02<00:18, 23.74it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  11%|█▏        | 57/500 [00:02<00:19, 23.11it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  12%|█▏        | 60/500 [00:02<00:19, 22.97it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  13%|█▎        | 63/500 [00:02<00:19, 22.21it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  13%|█▎        | 66/500 [00:02<00:20, 21.42it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  14%|█▍        | 69/500 [00:02<00:19, 21.55it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  14%|█▍        | 72/500 [00:03<00:20, 21.29it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  15%|█▌        | 75/500 [00:03<00:20, 20.99it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  16%|█▌        | 78/500 [00:03<00:20, 20.67it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  16%|█▌        | 81/500 [00:03<00:19, 21.08it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  17%|█▋        | 84/500 [00:03<00:19, 21.10it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  17%|█▋        | 87/500 [00:03<00:19, 21.13it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  18%|█▊        | 90/500 [00:03<00:19, 20.86it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  19%|█▊        | 93/500 [00:04<00:18, 21.43it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  19%|█▉        | 96/500 [00:04<00:18, 21.54it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  20%|█▉        | 99/500 [00:04<00:18, 21.15it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  20%|██        | 102/500 [00:04<00:18, 21.02it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  21%|██        | 105/500 [00:04<00:19, 20.68it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  22%|██▏       | 108/500 [00:04<00:19, 20.38it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  22%|██▏       | 111/500 [00:04<00:18, 20.55it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  23%|██▎       | 114/500 [00:05<00:18, 20.69it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  23%|██▎       | 117/500 [00:05<00:18, 20.63it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  24%|██▍       | 120/500 [00:05<00:18, 20.55it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  25%|██▍       | 123/500 [00:05<00:18, 20.35it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  25%|██▌       | 126/500 [00:05<00:18, 20.01it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  26%|██▌       | 129/500 [00:05<00:18, 19.80it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  26%|██▋       | 132/500 [00:06<00:18, 20.26it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  27%|██▋       | 135/500 [00:06<00:17, 20.39it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  28%|██▊       | 138/500 [00:06<00:17, 20.15it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  28%|██▊       | 141/500 [00:06<00:17, 20.04it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  29%|██▉       | 144/500 [00:06<00:17, 20.20it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  29%|██▉       | 147/500 [00:06<00:17, 20.01it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  30%|███       | 150/500 [00:06<00:17, 20.41it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  31%|███       | 153/500 [00:07<00:17, 20.35it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  31%|███       | 156/500 [00:07<00:16, 20.27it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  32%|███▏      | 159/500 [00:07<00:16, 20.31it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  32%|███▏      | 162/500 [00:07<00:16, 20.47it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  33%|███▎      | 165/500 [00:07<00:16, 20.39it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  34%|███▎      | 168/500 [00:07<00:16, 19.63it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  34%|███▍      | 170/500 [00:07<00:16, 19.47it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  34%|███▍      | 172/500 [00:08<00:16, 19.41it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  35%|███▍      | 174/500 [00:08<00:16, 19.47it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  35%|███▌      | 176/500 [00:08<00:16, 19.24it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  36%|███▌      | 178/500 [00:08<00:17, 18.94it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  36%|███▌      | 180/500 [00:08<00:17, 18.69it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  36%|███▋      | 182/500 [00:08<00:17, 18.40it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  37%|███▋      | 184/500 [00:08<00:17, 18.57it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  37%|███▋      | 186/500 [00:08<00:16, 18.65it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  38%|███▊      | 188/500 [00:08<00:16, 18.42it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  38%|███▊      | 190/500 [00:08<00:16, 18.72it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  38%|███▊      | 192/500 [00:09<00:16, 18.45it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  39%|███▉      | 194/500 [00:09<00:16, 18.43it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  39%|███▉      | 196/500 [00:09<00:16, 18.37it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  40%|███▉      | 198/500 [00:09<00:16, 18.38it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  40%|████      | 200/500 [00:09<00:16, 18.04it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  40%|████      | 202/500 [00:09<00:16, 18.14it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  41%|████      | 204/500 [00:09<00:16, 17.87it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  41%|████      | 206/500 [00:09<00:16, 17.82it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  42%|████▏     | 208/500 [00:10<00:16, 17.21it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  42%|████▏     | 210/500 [00:10<00:18, 16.02it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  42%|████▏     | 212/500 [00:10<00:17, 16.45it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  43%|████▎     | 214/500 [00:10<00:16, 17.21it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  43%|████▎     | 216/500 [00:10<00:16, 17.58it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  44%|████▎     | 218/500 [00:10<00:16, 17.61it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  44%|████▍     | 220/500 [00:10<00:15, 17.79it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  44%|████▍     | 222/500 [00:10<00:15, 17.74it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  45%|████▍     | 224/500 [00:10<00:15, 18.13it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  45%|████▌     | 226/500 [00:11<00:15, 18.21it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  46%|████▌     | 228/500 [00:11<00:14, 18.44it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  46%|████▌     | 230/500 [00:11<00:14, 18.13it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  46%|████▋     | 232/500 [00:11<00:15, 17.85it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  47%|████▋     | 234/500 [00:11<00:14, 17.88it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  47%|████▋     | 236/500 [00:11<00:14, 17.74it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  48%|████▊     | 238/500 [00:11<00:14, 17.58it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  48%|████▊     | 240/500 [00:11<00:15, 17.06it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  48%|████▊     | 242/500 [00:11<00:15, 17.13it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  49%|████▉     | 244/500 [00:12<00:14, 17.33it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  49%|████▉     | 246/500 [00:12<00:14, 17.51it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  50%|████▉     | 248/500 [00:12<00:14, 16.99it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  50%|█████     | 250/500 [00:12<00:15, 16.24it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  50%|█████     | 252/500 [00:12<00:14, 16.59it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  51%|█████     | 254/500 [00:12<00:14, 17.25it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  51%|█████     | 256/500 [00:12<00:14, 16.69it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  52%|█████▏    | 258/500 [00:12<00:14, 16.42it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  52%|█████▏    | 260/500 [00:13<00:14, 16.47it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  52%|█████▏    | 262/500 [00:13<00:14, 16.46it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  53%|█████▎    | 264/500 [00:13<00:14, 16.30it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  53%|█████▎    | 266/500 [00:13<00:14, 16.47it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  54%|█████▎    | 268/500 [00:13<00:13, 16.63it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  54%|█████▍    | 270/500 [00:13<00:13, 17.13it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  54%|█████▍    | 272/500 [00:13<00:13, 17.39it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  55%|█████▍    | 274/500 [00:13<00:12, 17.43it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  55%|█████▌    | 276/500 [00:13<00:12, 17.90it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  56%|█████▌    | 278/500 [00:14<00:12, 17.83it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  56%|█████▌    | 280/500 [00:14<00:12, 17.63it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  56%|█████▋    | 282/500 [00:14<00:12, 17.59it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  57%|█████▋    | 284/500 [00:14<00:12, 17.64it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  57%|█████▋    | 286/500 [00:14<00:12, 17.75it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  58%|█████▊    | 288/500 [00:14<00:11, 17.73it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  58%|█████▊    | 290/500 [00:14<00:11, 17.58it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  58%|█████▊    | 292/500 [00:14<00:11, 17.54it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  59%|█████▉    | 294/500 [00:14<00:11, 17.62it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  59%|█████▉    | 296/500 [00:15<00:11, 17.69it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  60%|█████▉    | 298/500 [00:15<00:11, 17.85it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  60%|██████    | 300/500 [00:15<00:11, 17.75it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  60%|██████    | 302/500 [00:15<00:10, 18.03it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  61%|██████    | 304/500 [00:15<00:10, 18.15it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  61%|██████    | 306/500 [00:15<00:10, 18.41it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  62%|██████▏   | 308/500 [00:15<00:10, 18.03it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  62%|██████▏   | 310/500 [00:15<00:10, 17.72it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  62%|██████▏   | 312/500 [00:15<00:10, 17.65it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  63%|██████▎   | 314/500 [00:16<00:10, 17.21it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  63%|██████▎   | 316/500 [00:16<00:10, 17.27it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  64%|██████▎   | 318/500 [00:16<00:10, 17.43it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  64%|██████▍   | 320/500 [00:16<00:10, 17.71it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  64%|██████▍   | 322/500 [00:16<00:10, 17.77it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  65%|██████▍   | 324/500 [00:16<00:09, 17.80it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  65%|██████▌   | 326/500 [00:16<00:09, 18.11it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  66%|██████▌   | 328/500 [00:16<00:09, 18.10it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  66%|██████▌   | 330/500 [00:16<00:09, 18.24it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  66%|██████▋   | 332/500 [00:17<00:09, 18.13it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  67%|██████▋   | 334/500 [00:17<00:09, 18.06it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  67%|██████▋   | 336/500 [00:17<00:09, 18.11it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  68%|██████▊   | 338/500 [00:17<00:08, 18.22it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  68%|██████▊   | 340/500 [00:17<00:08, 18.33it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  68%|██████▊   | 342/500 [00:17<00:08, 18.22it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  69%|██████▉   | 344/500 [00:17<00:08, 17.80it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  69%|██████▉   | 346/500 [00:17<00:08, 17.14it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  70%|██████▉   | 348/500 [00:18<00:08, 17.11it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  70%|███████   | 350/500 [00:18<00:08, 16.92it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  70%|███████   | 352/500 [00:18<00:08, 17.15it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  71%|███████   | 354/500 [00:18<00:08, 17.05it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  71%|███████   | 356/500 [00:18<00:08, 17.24it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  72%|███████▏  | 358/500 [00:18<00:08, 17.25it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  72%|███████▏  | 360/500 [00:18<00:08, 17.15it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  72%|███████▏  | 362/500 [00:18<00:08, 16.94it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  73%|███████▎  | 364/500 [00:18<00:07, 17.05it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  73%|███████▎  | 366/500 [00:19<00:07, 17.15it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  74%|███████▎  | 368/500 [00:19<00:07, 16.88it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  74%|███████▍  | 370/500 [00:19<00:07, 16.60it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  74%|███████▍  | 372/500 [00:19<00:07, 16.82it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  75%|███████▍  | 374/500 [00:19<00:07, 16.80it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  75%|███████▌  | 376/500 [00:19<00:07, 16.89it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  76%|███████▌  | 378/500 [00:19<00:07, 16.83it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  76%|███████▌  | 380/500 [00:19<00:07, 17.01it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  76%|███████▋  | 382/500 [00:20<00:06, 17.30it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  77%|███████▋  | 384/500 [00:20<00:06, 17.27it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  77%|███████▋  | 386/500 [00:20<00:06, 17.47it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  78%|███████▊  | 388/500 [00:20<00:06, 17.30it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  78%|███████▊  | 390/500 [00:20<00:06, 17.03it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  78%|███████▊  | 392/500 [00:20<00:06, 17.09it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  79%|███████▉  | 394/500 [00:20<00:06, 17.40it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  79%|███████▉  | 396/500 [00:20<00:05, 17.42it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  80%|███████▉  | 398/500 [00:20<00:05, 17.53it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  80%|████████  | 400/500 [00:21<00:05, 17.31it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  80%|████████  | 402/500 [00:21<00:05, 17.32it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  81%|████████  | 404/500 [00:21<00:05, 17.26it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  81%|████████  | 406/500 [00:21<00:05, 17.14it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  82%|████████▏ | 408/500 [00:21<00:05, 17.12it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  82%|████████▏ | 410/500 [00:21<00:05, 17.38it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  82%|████████▏ | 412/500 [00:21<00:05, 17.11it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  83%|████████▎ | 414/500 [00:21<00:05, 16.79it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Searching with note10 constraint (40):  83%|████████▎ | 416/500 [00:21<00:04, 17.01it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  84%|████████▎ | 418/500 [00:22<00:05, 16.26it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  84%|████████▍ | 420/500 [00:22<00:04, 16.52it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  84%|████████▍ | 422/500 [00:22<00:04, 16.72it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  85%|████████▍ | 424/500 [00:22<00:04, 17.11it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  85%|████████▌ | 426/500 [00:22<00:04, 17.67it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  86%|████████▌ | 428/500 [00:22<00:04, 17.92it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  86%|████████▌ | 430/500 [00:22<00:03, 18.19it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  86%|████████▋ | 432/500 [00:22<00:03, 17.78it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  87%|████████▋ | 434/500 [00:23<00:03, 17.95it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  87%|████████▋ | 436/500 [00:23<00:03, 17.63it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  88%|████████▊ | 438/500 [00:23<00:03, 17.70it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  88%|████████▊ | 440/500 [00:23<00:03, 17.75it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  88%|████████▊ | 442/500 [00:23<00:03, 17.92it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  89%|████████▉ | 444/500 [00:23<00:03, 17.67it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  89%|████████▉ | 446/500 [00:23<00:03, 17.77it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  90%|████████▉ | 448/500 [00:23<00:02, 17.56it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  90%|█████████ | 450/500 [00:23<00:03, 16.62it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  90%|█████████ | 452/500 [00:24<00:02, 17.02it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  91%|█████████ | 454/500 [00:24<00:02, 17.52it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  91%|█████████ | 456/500 [00:24<00:02, 17.90it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  92%|█████████▏| 458/500 [00:24<00:02, 18.19it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  92%|█████████▏| 461/500 [00:24<00:02, 18.90it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  93%|█████████▎| 463/500 [00:24<00:01, 19.09it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  93%|█████████▎| 465/500 [00:24<00:01, 19.30it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  93%|█████████▎| 467/500 [00:24<00:01, 19.00it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  94%|█████████▍| 469/500 [00:24<00:01, 19.21it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  94%|█████████▍| 471/500 [00:25<00:01, 18.75it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  95%|█████████▍| 473/500 [00:25<00:01, 18.78it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  95%|█████████▌| 475/500 [00:25<00:01, 18.85it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  95%|█████████▌| 477/500 [00:25<00:01, 19.02it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  96%|█████████▌| 479/500 [00:25<00:01, 18.84it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  96%|█████████▌| 481/500 [00:25<00:01, 18.84it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  97%|█████████▋| 483/500 [00:25<00:00, 19.13it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  97%|█████████▋| 485/500 [00:25<00:00, 19.30it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  97%|█████████▋| 487/500 [00:25<00:00, 18.86it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  98%|█████████▊| 489/500 [00:25<00:00, 18.94it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  98%|█████████▊| 491/500 [00:26<00:00, 18.85it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  99%|█████████▉| 494/500 [00:26<00:00, 19.39it/s]\u001b[A\n",
      "Searching with note10 constraint (40):  99%|█████████▉| 497/500 [00:26<00:00, 19.81it/s]\u001b[A\n",
      "Searching with note10 constraint (40): 100%|██████████| 500/500 [00:26<00:00, 18.84it/s]\u001b[A\n",
      "\n",
      "Searching with note10 constraint (45):   0%|          | 0/500 [00:00<?, ?it/s]\u001b[A\n",
      "Searching with note10 constraint (45):   1%|          | 3/500 [00:00<00:21, 23.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 40.00 msIt achieves 83.62% predicted accuracy with 39.85 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E4.0, K3), Identity)\n",
      "(SE(O48, E3.0, K7), None)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E4.0, K5), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E3.0, K7), Identity)\n",
      "((O96, E6.0, K7), Identity)\n",
      "((O96, E6.0, K5), Identity)\n",
      "(SE(O136, E4.0, K5), None)\n",
      "(SE(O136, E6.0, K5), Identity)\n",
      "(SE(O136, E6.0, K7), Identity)\n",
      "(SE(O136, E3.0, K3), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  26.576290130615234\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Searching with note10 constraint (45):   1%|          | 5/500 [00:00<00:22, 21.75it/s]\u001b[A\n",
      "Searching with note10 constraint (45):   1%|▏         | 7/500 [00:00<00:24, 20.18it/s]\u001b[A\n",
      "Searching with note10 constraint (45):   2%|▏         | 9/500 [00:00<00:25, 19.32it/s]\u001b[A\n",
      "Searching with note10 constraint (45):   2%|▏         | 11/500 [00:00<00:25, 19.32it/s]\u001b[A\n",
      "Searching with note10 constraint (45):   3%|▎         | 13/500 [00:00<00:25, 19.47it/s]\u001b[A\n",
      "Searching with note10 constraint (45):   3%|▎         | 16/500 [00:00<00:24, 19.94it/s]\u001b[A\n",
      "Searching with note10 constraint (45):   4%|▍         | 19/500 [00:00<00:23, 20.26it/s]\u001b[A\n",
      "Searching with note10 constraint (45):   4%|▍         | 22/500 [00:01<00:23, 20.64it/s]\u001b[A\n",
      "Searching with note10 constraint (45):   5%|▌         | 25/500 [00:01<00:22, 21.12it/s]\u001b[A\n",
      "Searching with note10 constraint (45):   6%|▌         | 28/500 [00:01<00:23, 20.48it/s]\u001b[A\n",
      "Searching with note10 constraint (45):   6%|▌         | 31/500 [00:01<00:24, 19.25it/s]\u001b[A\n",
      "Searching with note10 constraint (45):   7%|▋         | 34/500 [00:01<00:23, 19.99it/s]\u001b[A\n",
      "Searching with note10 constraint (45):   7%|▋         | 37/500 [00:01<00:22, 20.73it/s]\u001b[A\n",
      "Searching with note10 constraint (45):   8%|▊         | 40/500 [00:01<00:21, 21.45it/s]\u001b[A\n",
      "Searching with note10 constraint (45):   9%|▊         | 43/500 [00:02<00:20, 21.99it/s]\u001b[A\n",
      "Searching with note10 constraint (45):   9%|▉         | 46/500 [00:02<00:20, 22.54it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  10%|▉         | 49/500 [00:02<00:19, 22.67it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  10%|█         | 52/500 [00:02<00:19, 23.06it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  11%|█         | 55/500 [00:02<00:19, 22.91it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  12%|█▏        | 58/500 [00:02<00:19, 23.02it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  12%|█▏        | 61/500 [00:02<00:19, 23.10it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  13%|█▎        | 64/500 [00:02<00:18, 23.16it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  13%|█▎        | 67/500 [00:03<00:18, 23.22it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  14%|█▍        | 70/500 [00:03<00:18, 23.03it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  15%|█▍        | 73/500 [00:03<00:18, 23.09it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  15%|█▌        | 76/500 [00:03<00:18, 23.18it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  16%|█▌        | 79/500 [00:03<00:18, 23.25it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  16%|█▋        | 82/500 [00:03<00:17, 23.47it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  17%|█▋        | 85/500 [00:03<00:17, 23.36it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  18%|█▊        | 88/500 [00:04<00:17, 23.34it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  18%|█▊        | 91/500 [00:04<00:17, 23.57it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  19%|█▉        | 94/500 [00:04<00:17, 23.71it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  19%|█▉        | 97/500 [00:04<00:17, 23.32it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  20%|██        | 100/500 [00:04<00:17, 23.36it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  21%|██        | 103/500 [00:04<00:17, 23.25it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  21%|██        | 106/500 [00:04<00:16, 23.53it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  22%|██▏       | 109/500 [00:04<00:16, 23.17it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  22%|██▏       | 112/500 [00:05<00:16, 23.36it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  23%|██▎       | 115/500 [00:05<00:16, 23.23it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  24%|██▎       | 118/500 [00:05<00:16, 23.29it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  24%|██▍       | 121/500 [00:05<00:16, 23.46it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  25%|██▍       | 124/500 [00:05<00:16, 23.40it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  25%|██▌       | 127/500 [00:05<00:15, 23.43it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  26%|██▌       | 130/500 [00:05<00:15, 23.26it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  27%|██▋       | 133/500 [00:05<00:15, 23.44it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  27%|██▋       | 136/500 [00:06<00:15, 23.63it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  28%|██▊       | 139/500 [00:06<00:15, 23.44it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  28%|██▊       | 142/500 [00:06<00:15, 23.43it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  29%|██▉       | 145/500 [00:06<00:15, 23.24it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  30%|██▉       | 148/500 [00:06<00:15, 22.74it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  30%|███       | 151/500 [00:06<00:15, 23.07it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  31%|███       | 154/500 [00:06<00:14, 23.26it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  31%|███▏      | 157/500 [00:06<00:14, 23.43it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  32%|███▏      | 160/500 [00:07<00:14, 23.49it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  33%|███▎      | 163/500 [00:07<00:14, 23.82it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  33%|███▎      | 166/500 [00:07<00:13, 23.92it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  34%|███▍      | 169/500 [00:07<00:14, 23.47it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  34%|███▍      | 172/500 [00:07<00:13, 23.51it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  35%|███▌      | 175/500 [00:07<00:13, 23.90it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  36%|███▌      | 178/500 [00:07<00:13, 23.56it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  36%|███▌      | 181/500 [00:07<00:13, 23.63it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  37%|███▋      | 184/500 [00:08<00:13, 23.74it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  37%|███▋      | 187/500 [00:08<00:13, 23.87it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  38%|███▊      | 190/500 [00:08<00:13, 23.59it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  39%|███▊      | 193/500 [00:08<00:13, 23.50it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  39%|███▉      | 196/500 [00:08<00:12, 23.50it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  40%|███▉      | 199/500 [00:08<00:12, 23.55it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  40%|████      | 202/500 [00:08<00:12, 23.50it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  41%|████      | 205/500 [00:08<00:12, 23.86it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  42%|████▏     | 208/500 [00:09<00:12, 24.13it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  42%|████▏     | 211/500 [00:09<00:11, 24.26it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  43%|████▎     | 214/500 [00:09<00:11, 24.02it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  43%|████▎     | 217/500 [00:09<00:11, 24.14it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  44%|████▍     | 220/500 [00:09<00:11, 24.10it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  45%|████▍     | 223/500 [00:09<00:11, 24.01it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  45%|████▌     | 226/500 [00:09<00:11, 23.96it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  46%|████▌     | 229/500 [00:09<00:11, 23.95it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  46%|████▋     | 232/500 [00:10<00:11, 23.95it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  47%|████▋     | 235/500 [00:10<00:10, 24.11it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  48%|████▊     | 238/500 [00:10<00:10, 23.93it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  48%|████▊     | 241/500 [00:10<00:10, 23.92it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  49%|████▉     | 244/500 [00:10<00:10, 23.74it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  49%|████▉     | 247/500 [00:10<00:10, 24.07it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  50%|█████     | 250/500 [00:10<00:10, 23.88it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  51%|█████     | 253/500 [00:10<00:10, 24.15it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  51%|█████     | 256/500 [00:11<00:09, 24.67it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  52%|█████▏    | 259/500 [00:11<00:09, 24.75it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  52%|█████▏    | 262/500 [00:11<00:09, 24.87it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  53%|█████▎    | 265/500 [00:11<00:09, 24.80it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  54%|█████▎    | 268/500 [00:11<00:09, 24.01it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  54%|█████▍    | 271/500 [00:11<00:09, 24.19it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  55%|█████▍    | 274/500 [00:11<00:09, 24.34it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  55%|█████▌    | 277/500 [00:11<00:09, 24.66it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  56%|█████▌    | 280/500 [00:12<00:08, 24.51it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  57%|█████▋    | 283/500 [00:12<00:08, 24.55it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  57%|█████▋    | 286/500 [00:12<00:08, 24.39it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  58%|█████▊    | 289/500 [00:12<00:08, 24.59it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  58%|█████▊    | 292/500 [00:12<00:08, 24.67it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  59%|█████▉    | 295/500 [00:12<00:08, 24.84it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  60%|█████▉    | 298/500 [00:12<00:08, 24.65it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  60%|██████    | 301/500 [00:12<00:08, 24.58it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  61%|██████    | 304/500 [00:13<00:07, 24.72it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  61%|██████▏   | 307/500 [00:13<00:07, 24.70it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  62%|██████▏   | 310/500 [00:13<00:07, 24.70it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  63%|██████▎   | 313/500 [00:13<00:07, 24.79it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  63%|██████▎   | 316/500 [00:13<00:07, 24.78it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  64%|██████▍   | 319/500 [00:13<00:07, 24.87it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  64%|██████▍   | 322/500 [00:13<00:07, 24.88it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  65%|██████▌   | 325/500 [00:13<00:07, 24.73it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  66%|██████▌   | 328/500 [00:14<00:07, 24.31it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  66%|██████▌   | 331/500 [00:14<00:06, 24.55it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  67%|██████▋   | 334/500 [00:14<00:06, 24.36it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  67%|██████▋   | 337/500 [00:14<00:06, 24.06it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  68%|██████▊   | 340/500 [00:14<00:06, 24.02it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  69%|██████▊   | 343/500 [00:14<00:06, 24.31it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  69%|██████▉   | 346/500 [00:14<00:06, 24.29it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  70%|██████▉   | 349/500 [00:14<00:06, 24.79it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  70%|███████   | 352/500 [00:15<00:05, 25.27it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  71%|███████   | 355/500 [00:15<00:05, 25.64it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  72%|███████▏  | 358/500 [00:15<00:05, 25.64it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  72%|███████▏  | 361/500 [00:15<00:05, 25.47it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  73%|███████▎  | 364/500 [00:15<00:05, 25.54it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  73%|███████▎  | 367/500 [00:15<00:05, 25.18it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  74%|███████▍  | 370/500 [00:15<00:05, 25.50it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  75%|███████▍  | 373/500 [00:15<00:05, 25.19it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  75%|███████▌  | 376/500 [00:15<00:04, 24.97it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  76%|███████▌  | 379/500 [00:16<00:04, 24.87it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  76%|███████▋  | 382/500 [00:16<00:04, 24.80it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  77%|███████▋  | 385/500 [00:16<00:04, 24.66it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  78%|███████▊  | 388/500 [00:16<00:04, 24.61it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  78%|███████▊  | 391/500 [00:16<00:04, 24.46it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  79%|███████▉  | 394/500 [00:16<00:04, 24.40it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  79%|███████▉  | 397/500 [00:16<00:04, 23.90it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  80%|████████  | 400/500 [00:16<00:04, 24.14it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  81%|████████  | 403/500 [00:17<00:03, 24.45it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  81%|████████  | 406/500 [00:17<00:03, 25.03it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  82%|████████▏ | 409/500 [00:17<00:03, 25.04it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  82%|████████▏ | 412/500 [00:17<00:03, 24.85it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  83%|████████▎ | 415/500 [00:17<00:03, 24.93it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  84%|████████▎ | 418/500 [00:17<00:03, 25.15it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  84%|████████▍ | 421/500 [00:17<00:03, 25.42it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  85%|████████▍ | 424/500 [00:17<00:02, 25.64it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  85%|████████▌ | 427/500 [00:18<00:02, 25.52it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  86%|████████▌ | 430/500 [00:18<00:02, 25.84it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  87%|████████▋ | 433/500 [00:18<00:02, 25.93it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  87%|████████▋ | 436/500 [00:18<00:02, 25.92it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  88%|████████▊ | 439/500 [00:18<00:02, 25.57it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  88%|████████▊ | 442/500 [00:18<00:02, 25.53it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  89%|████████▉ | 445/500 [00:18<00:02, 25.82it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  90%|████████▉ | 448/500 [00:18<00:02, 25.91it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  90%|█████████ | 451/500 [00:18<00:01, 26.06it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  91%|█████████ | 454/500 [00:19<00:01, 26.14it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  91%|█████████▏| 457/500 [00:19<00:01, 25.78it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  92%|█████████▏| 460/500 [00:19<00:01, 26.26it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  93%|█████████▎| 463/500 [00:19<00:01, 26.34it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  93%|█████████▎| 466/500 [00:19<00:01, 26.34it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  94%|█████████▍| 469/500 [00:19<00:01, 25.98it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  94%|█████████▍| 472/500 [00:19<00:01, 26.13it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  95%|█████████▌| 475/500 [00:19<00:00, 26.14it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  96%|█████████▌| 478/500 [00:19<00:00, 26.14it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  96%|█████████▌| 481/500 [00:20<00:00, 26.24it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  97%|█████████▋| 484/500 [00:20<00:00, 26.05it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  97%|█████████▋| 487/500 [00:20<00:00, 26.00it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  98%|█████████▊| 490/500 [00:20<00:00, 25.78it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  99%|█████████▊| 493/500 [00:20<00:00, 25.67it/s]\u001b[A\n",
      "Searching with note10 constraint (45):  99%|█████████▉| 496/500 [00:20<00:00, 25.45it/s]\u001b[A\n",
      "Searching with note10 constraint (45): 100%|██████████| 500/500 [00:20<00:00, 24.01it/s]\u001b[A\n",
      "\n",
      "Searching with note10 constraint (50):   0%|          | 0/500 [00:00<?, ?it/s]\u001b[A\n",
      "Searching with note10 constraint (50):   1%|          | 3/500 [00:00<00:18, 27.40it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 45.00 msIt achieves 84.29% predicted accuracy with 43.99 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E4.0, K3), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E4.0, K3), Identity)\n",
      "(SE(O48, E6.0, K5), None)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E3.0, K7), Identity)\n",
      "((O96, E4.0, K3), Identity)\n",
      "((O96, E6.0, K7), Identity)\n",
      "(SE(O136, E6.0, K7), None)\n",
      "(SE(O136, E4.0, K7), Identity)\n",
      "(SE(O136, E6.0, K5), Identity)\n",
      "(SE(O136, E6.0, K5), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  20.8565890789032\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Searching with note10 constraint (50):   1%|          | 6/500 [00:00<00:18, 26.19it/s]\u001b[A\n",
      "Searching with note10 constraint (50):   2%|▏         | 9/500 [00:00<00:19, 25.56it/s]\u001b[A\n",
      "Searching with note10 constraint (50):   2%|▏         | 12/500 [00:00<00:19, 25.57it/s]\u001b[A\n",
      "Searching with note10 constraint (50):   3%|▎         | 15/500 [00:00<00:19, 25.38it/s]\u001b[A\n",
      "Searching with note10 constraint (50):   4%|▎         | 18/500 [00:00<00:18, 25.38it/s]\u001b[A\n",
      "Searching with note10 constraint (50):   4%|▍         | 21/500 [00:00<00:19, 24.77it/s]\u001b[A\n",
      "Searching with note10 constraint (50):   5%|▍         | 24/500 [00:00<00:19, 24.40it/s]\u001b[A\n",
      "Searching with note10 constraint (50):   5%|▌         | 27/500 [00:01<00:19, 24.41it/s]\u001b[A\n",
      "Searching with note10 constraint (50):   6%|▌         | 30/500 [00:01<00:18, 24.89it/s]\u001b[A\n",
      "Searching with note10 constraint (50):   7%|▋         | 33/500 [00:01<00:18, 25.03it/s]\u001b[A\n",
      "Searching with note10 constraint (50):   7%|▋         | 36/500 [00:01<00:18, 25.51it/s]\u001b[A\n",
      "Searching with note10 constraint (50):   8%|▊         | 39/500 [00:01<00:18, 25.15it/s]\u001b[A\n",
      "Searching with note10 constraint (50):   8%|▊         | 42/500 [00:01<00:17, 25.56it/s]\u001b[A\n",
      "Searching with note10 constraint (50):   9%|▉         | 45/500 [00:01<00:17, 25.76it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  10%|▉         | 48/500 [00:01<00:17, 25.99it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  10%|█         | 51/500 [00:02<00:17, 26.12it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  11%|█         | 54/500 [00:02<00:17, 25.87it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  11%|█▏        | 57/500 [00:02<00:17, 25.96it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  12%|█▏        | 60/500 [00:02<00:17, 25.87it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  13%|█▎        | 63/500 [00:02<00:16, 25.90it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  13%|█▎        | 66/500 [00:02<00:16, 25.70it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  14%|█▍        | 69/500 [00:02<00:16, 26.03it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  14%|█▍        | 72/500 [00:02<00:16, 26.30it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  15%|█▌        | 75/500 [00:02<00:16, 26.21it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  16%|█▌        | 78/500 [00:03<00:16, 25.91it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  16%|█▌        | 81/500 [00:03<00:16, 26.07it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  17%|█▋        | 84/500 [00:03<00:16, 25.81it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  17%|█▋        | 87/500 [00:03<00:15, 25.88it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  18%|█▊        | 90/500 [00:03<00:15, 25.90it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  19%|█▊        | 93/500 [00:03<00:15, 26.06it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  19%|█▉        | 96/500 [00:03<00:15, 26.25it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  20%|█▉        | 99/500 [00:03<00:15, 26.15it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  20%|██        | 102/500 [00:03<00:15, 26.33it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  21%|██        | 105/500 [00:04<00:15, 26.23it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  22%|██▏       | 108/500 [00:04<00:14, 26.34it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  22%|██▏       | 111/500 [00:04<00:14, 26.34it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  23%|██▎       | 114/500 [00:04<00:14, 26.17it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  23%|██▎       | 117/500 [00:04<00:14, 26.53it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  24%|██▍       | 120/500 [00:04<00:14, 26.31it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  25%|██▍       | 123/500 [00:04<00:14, 25.50it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  25%|██▌       | 126/500 [00:04<00:14, 25.42it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  26%|██▌       | 129/500 [00:05<00:14, 25.40it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  26%|██▋       | 132/500 [00:05<00:14, 25.36it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  27%|██▋       | 135/500 [00:05<00:14, 25.59it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  28%|██▊       | 138/500 [00:05<00:14, 25.50it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  28%|██▊       | 141/500 [00:05<00:14, 25.33it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  29%|██▉       | 144/500 [00:05<00:14, 24.84it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  29%|██▉       | 147/500 [00:05<00:14, 24.69it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  30%|███       | 150/500 [00:05<00:14, 24.90it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  31%|███       | 153/500 [00:05<00:14, 24.16it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  31%|███       | 156/500 [00:06<00:14, 24.06it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  32%|███▏      | 159/500 [00:06<00:13, 24.40it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  32%|███▏      | 162/500 [00:06<00:13, 24.77it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  33%|███▎      | 165/500 [00:06<00:13, 24.73it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  34%|███▎      | 168/500 [00:06<00:13, 24.75it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  34%|███▍      | 171/500 [00:06<00:13, 25.08it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  35%|███▍      | 174/500 [00:06<00:13, 24.98it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  35%|███▌      | 177/500 [00:06<00:12, 25.29it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  36%|███▌      | 180/500 [00:07<00:12, 25.14it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  37%|███▋      | 183/500 [00:07<00:12, 25.17it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  37%|███▋      | 186/500 [00:07<00:12, 25.62it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  38%|███▊      | 189/500 [00:07<00:12, 25.67it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  38%|███▊      | 192/500 [00:07<00:11, 25.72it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  39%|███▉      | 195/500 [00:07<00:11, 25.55it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  40%|███▉      | 198/500 [00:07<00:11, 25.31it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  40%|████      | 201/500 [00:07<00:11, 25.41it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  41%|████      | 204/500 [00:08<00:11, 25.65it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  41%|████▏     | 207/500 [00:08<00:11, 25.80it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  42%|████▏     | 210/500 [00:08<00:11, 25.69it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  43%|████▎     | 213/500 [00:08<00:11, 25.45it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  43%|████▎     | 216/500 [00:08<00:11, 25.38it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  44%|████▍     | 219/500 [00:08<00:11, 25.47it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  44%|████▍     | 222/500 [00:08<00:11, 25.25it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  45%|████▌     | 225/500 [00:08<00:10, 25.28it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  46%|████▌     | 228/500 [00:08<00:10, 25.33it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  46%|████▌     | 231/500 [00:09<00:10, 25.06it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  47%|████▋     | 234/500 [00:09<00:10, 24.37it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  47%|████▋     | 237/500 [00:09<00:11, 23.74it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  48%|████▊     | 240/500 [00:09<00:10, 24.09it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  49%|████▊     | 243/500 [00:09<00:10, 24.39it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  49%|████▉     | 246/500 [00:09<00:10, 24.52it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  50%|████▉     | 249/500 [00:09<00:10, 24.71it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  50%|█████     | 252/500 [00:09<00:09, 24.97it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  51%|█████     | 255/500 [00:10<00:09, 24.94it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  52%|█████▏    | 258/500 [00:10<00:09, 24.88it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  52%|█████▏    | 261/500 [00:10<00:09, 25.07it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  53%|█████▎    | 264/500 [00:10<00:09, 25.07it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  53%|█████▎    | 267/500 [00:10<00:09, 25.26it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  54%|█████▍    | 270/500 [00:10<00:09, 25.35it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  55%|█████▍    | 273/500 [00:10<00:09, 24.97it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  55%|█████▌    | 276/500 [00:10<00:09, 24.68it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  56%|█████▌    | 279/500 [00:11<00:09, 24.23it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  56%|█████▋    | 282/500 [00:11<00:09, 23.75it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  57%|█████▋    | 285/500 [00:11<00:08, 24.09it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  58%|█████▊    | 288/500 [00:11<00:08, 24.60it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  58%|█████▊    | 291/500 [00:11<00:08, 24.66it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  59%|█████▉    | 294/500 [00:11<00:08, 24.87it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  59%|█████▉    | 297/500 [00:11<00:08, 25.12it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  60%|██████    | 300/500 [00:11<00:08, 24.95it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  61%|██████    | 303/500 [00:11<00:07, 25.30it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  61%|██████    | 306/500 [00:12<00:07, 25.31it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  62%|██████▏   | 309/500 [00:12<00:07, 25.39it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  62%|██████▏   | 312/500 [00:12<00:07, 25.44it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  63%|██████▎   | 315/500 [00:12<00:07, 25.74it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  64%|██████▎   | 318/500 [00:12<00:07, 25.73it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  64%|██████▍   | 321/500 [00:12<00:07, 25.49it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  65%|██████▍   | 324/500 [00:12<00:06, 25.44it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  65%|██████▌   | 327/500 [00:12<00:06, 25.34it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  66%|██████▌   | 330/500 [00:13<00:06, 25.56it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  67%|██████▋   | 333/500 [00:13<00:06, 25.39it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  67%|██████▋   | 336/500 [00:13<00:06, 25.21it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  68%|██████▊   | 339/500 [00:13<00:06, 25.32it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  68%|██████▊   | 342/500 [00:13<00:06, 25.23it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  69%|██████▉   | 345/500 [00:13<00:06, 25.44it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  70%|██████▉   | 348/500 [00:13<00:05, 25.54it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  70%|███████   | 351/500 [00:13<00:05, 25.67it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  71%|███████   | 354/500 [00:13<00:05, 25.85it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  71%|███████▏  | 357/500 [00:14<00:05, 25.76it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  72%|███████▏  | 360/500 [00:14<00:05, 25.73it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  73%|███████▎  | 363/500 [00:14<00:05, 25.47it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  73%|███████▎  | 366/500 [00:14<00:05, 25.52it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  74%|███████▍  | 369/500 [00:14<00:05, 25.64it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  74%|███████▍  | 372/500 [00:14<00:05, 25.43it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  75%|███████▌  | 375/500 [00:14<00:04, 25.65it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  76%|███████▌  | 378/500 [00:14<00:04, 26.08it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  76%|███████▌  | 381/500 [00:15<00:04, 25.80it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  77%|███████▋  | 384/500 [00:15<00:04, 25.47it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  77%|███████▋  | 387/500 [00:15<00:04, 25.53it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  78%|███████▊  | 390/500 [00:15<00:04, 25.16it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  79%|███████▊  | 393/500 [00:15<00:04, 24.69it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  79%|███████▉  | 396/500 [00:15<00:04, 25.20it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  80%|███████▉  | 399/500 [00:15<00:04, 25.10it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  80%|████████  | 402/500 [00:15<00:03, 25.33it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  81%|████████  | 405/500 [00:16<00:03, 24.87it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  82%|████████▏ | 408/500 [00:16<00:03, 24.38it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  82%|████████▏ | 411/500 [00:16<00:03, 23.69it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  83%|████████▎ | 414/500 [00:16<00:03, 23.69it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  83%|████████▎ | 417/500 [00:16<00:03, 24.16it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  84%|████████▍ | 420/500 [00:16<00:03, 23.93it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  85%|████████▍ | 423/500 [00:16<00:03, 24.10it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  85%|████████▌ | 426/500 [00:16<00:03, 24.04it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  86%|████████▌ | 429/500 [00:17<00:02, 24.27it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  86%|████████▋ | 432/500 [00:17<00:02, 24.71it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  87%|████████▋ | 435/500 [00:17<00:02, 24.71it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  88%|████████▊ | 438/500 [00:17<00:02, 24.70it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  88%|████████▊ | 441/500 [00:17<00:02, 24.87it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  89%|████████▉ | 444/500 [00:17<00:02, 25.05it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  89%|████████▉ | 447/500 [00:17<00:02, 25.01it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  90%|█████████ | 450/500 [00:17<00:01, 25.27it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  91%|█████████ | 453/500 [00:17<00:01, 25.31it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  91%|█████████ | 456/500 [00:18<00:01, 25.84it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  92%|█████████▏| 459/500 [00:18<00:01, 25.46it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  92%|█████████▏| 462/500 [00:18<00:01, 25.31it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  93%|█████████▎| 465/500 [00:18<00:01, 25.58it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  94%|█████████▎| 468/500 [00:18<00:01, 25.87it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  94%|█████████▍| 471/500 [00:18<00:01, 25.56it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  95%|█████████▍| 474/500 [00:18<00:01, 25.59it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  95%|█████████▌| 477/500 [00:18<00:00, 25.13it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  96%|█████████▌| 480/500 [00:19<00:00, 25.13it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  97%|█████████▋| 483/500 [00:19<00:00, 25.09it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  97%|█████████▋| 486/500 [00:19<00:00, 24.79it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  98%|█████████▊| 489/500 [00:19<00:00, 25.09it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  98%|█████████▊| 492/500 [00:19<00:00, 25.19it/s]\u001b[A\n",
      "Searching with note10 constraint (50):  99%|█████████▉| 495/500 [00:19<00:00, 25.41it/s]\u001b[A\n",
      "Searching with note10 constraint (50): 100%|██████████| 500/500 [00:19<00:00, 25.23it/s]\u001b[A\n",
      "\n",
      "Searching with note10 constraint (55):   0%|          | 0/500 [00:00<?, ?it/s]\u001b[A\n",
      "Searching with note10 constraint (55):   1%|          | 4/500 [00:00<00:16, 30.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 50.00 msIt achieves 84.35% predicted accuracy with 49.81 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E3.0, K5), None)\n",
      "((O32, E4.0, K5), Identity)\n",
      "((O32, E6.0, K3), Identity)\n",
      "(SE(O48, E4.0, K7), None)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "(SE(O48, E4.0, K5), Identity)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E3.0, K3), Identity)\n",
      "((O96, E4.0, K3), Identity)\n",
      "((O96, E6.0, K3), Identity)\n",
      "(SE(O136, E3.0, K7), None)\n",
      "(SE(O136, E6.0, K7), Identity)\n",
      "(SE(O136, E6.0, K5), Identity)\n",
      "(SE(O136, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K3), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E4.0, K5), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  19.848477125167847\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Searching with note10 constraint (55):   1%|▏         | 7/500 [00:00<00:17, 28.75it/s]\u001b[A\n",
      "Searching with note10 constraint (55):   2%|▏         | 10/500 [00:00<00:17, 27.64it/s]\u001b[A\n",
      "Searching with note10 constraint (55):   3%|▎         | 13/500 [00:00<00:18, 26.98it/s]\u001b[A\n",
      "Searching with note10 constraint (55):   3%|▎         | 16/500 [00:00<00:18, 26.87it/s]\u001b[A\n",
      "Searching with note10 constraint (55):   4%|▍         | 19/500 [00:00<00:18, 26.45it/s]\u001b[A\n",
      "Searching with note10 constraint (55):   4%|▍         | 22/500 [00:00<00:18, 25.39it/s]\u001b[A\n",
      "Searching with note10 constraint (55):   5%|▌         | 25/500 [00:00<00:18, 25.25it/s]\u001b[A\n",
      "Searching with note10 constraint (55):   6%|▌         | 28/500 [00:01<00:18, 25.92it/s]\u001b[A\n",
      "Searching with note10 constraint (55):   6%|▌         | 31/500 [00:01<00:17, 26.19it/s]\u001b[A\n",
      "Searching with note10 constraint (55):   7%|▋         | 34/500 [00:01<00:17, 25.90it/s]\u001b[A\n",
      "Searching with note10 constraint (55):   7%|▋         | 37/500 [00:01<00:18, 25.55it/s]\u001b[A\n",
      "Searching with note10 constraint (55):   8%|▊         | 40/500 [00:01<00:17, 25.84it/s]\u001b[A\n",
      "Searching with note10 constraint (55):   9%|▊         | 43/500 [00:01<00:17, 26.04it/s]\u001b[A\n",
      "Searching with note10 constraint (55):   9%|▉         | 46/500 [00:01<00:17, 26.45it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  10%|▉         | 49/500 [00:01<00:16, 26.53it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  10%|█         | 52/500 [00:01<00:16, 26.75it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  11%|█         | 55/500 [00:02<00:17, 26.08it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  12%|█▏        | 58/500 [00:02<00:16, 26.47it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  12%|█▏        | 61/500 [00:02<00:16, 26.60it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  13%|█▎        | 64/500 [00:02<00:16, 26.48it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  13%|█▎        | 67/500 [00:02<00:16, 26.74it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  14%|█▍        | 70/500 [00:02<00:15, 26.90it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  15%|█▍        | 73/500 [00:02<00:15, 27.12it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  15%|█▌        | 76/500 [00:02<00:15, 26.93it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  16%|█▌        | 79/500 [00:02<00:15, 26.70it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  16%|█▋        | 82/500 [00:03<00:15, 26.16it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  17%|█▋        | 85/500 [00:03<00:15, 26.27it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  18%|█▊        | 88/500 [00:03<00:15, 26.16it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  18%|█▊        | 91/500 [00:03<00:15, 26.26it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  19%|█▉        | 94/500 [00:03<00:15, 26.33it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  19%|█▉        | 97/500 [00:03<00:15, 26.21it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  20%|██        | 100/500 [00:03<00:15, 26.43it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  21%|██        | 103/500 [00:03<00:15, 26.30it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  21%|██        | 106/500 [00:04<00:14, 26.62it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  22%|██▏       | 109/500 [00:04<00:14, 26.75it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  22%|██▏       | 112/500 [00:04<00:14, 26.59it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  23%|██▎       | 115/500 [00:04<00:14, 26.13it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  24%|██▎       | 118/500 [00:04<00:14, 26.25it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  24%|██▍       | 121/500 [00:04<00:14, 26.39it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  25%|██▍       | 124/500 [00:04<00:14, 26.47it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  25%|██▌       | 127/500 [00:04<00:14, 26.22it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  26%|██▌       | 130/500 [00:04<00:14, 26.00it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  27%|██▋       | 133/500 [00:05<00:14, 26.01it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  27%|██▋       | 136/500 [00:05<00:14, 24.68it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  28%|██▊       | 139/500 [00:05<00:14, 25.19it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  28%|██▊       | 142/500 [00:05<00:14, 25.30it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  29%|██▉       | 145/500 [00:05<00:13, 25.41it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  30%|██▉       | 148/500 [00:05<00:13, 25.74it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  30%|███       | 151/500 [00:05<00:13, 26.03it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  31%|███       | 154/500 [00:05<00:13, 25.65it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  31%|███▏      | 157/500 [00:05<00:13, 26.18it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  32%|███▏      | 160/500 [00:06<00:13, 25.96it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  33%|███▎      | 163/500 [00:06<00:12, 25.93it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  33%|███▎      | 166/500 [00:06<00:13, 25.66it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  34%|███▍      | 169/500 [00:06<00:13, 25.03it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  34%|███▍      | 172/500 [00:06<00:12, 25.35it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  35%|███▌      | 175/500 [00:06<00:12, 25.72it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  36%|███▌      | 178/500 [00:06<00:12, 25.85it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  36%|███▌      | 181/500 [00:06<00:12, 25.70it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  37%|███▋      | 184/500 [00:07<00:12, 25.68it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  37%|███▋      | 187/500 [00:07<00:12, 26.04it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  38%|███▊      | 190/500 [00:07<00:11, 26.31it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  39%|███▊      | 193/500 [00:07<00:11, 26.68it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  39%|███▉      | 196/500 [00:07<00:11, 26.47it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  40%|███▉      | 199/500 [00:07<00:11, 26.53it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  40%|████      | 202/500 [00:07<00:11, 26.63it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  41%|████      | 205/500 [00:07<00:11, 26.69it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  42%|████▏     | 208/500 [00:07<00:11, 25.84it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  42%|████▏     | 211/500 [00:08<00:11, 25.99it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  43%|████▎     | 214/500 [00:08<00:11, 25.97it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  43%|████▎     | 217/500 [00:08<00:10, 26.25it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  44%|████▍     | 220/500 [00:08<00:10, 25.92it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  45%|████▍     | 223/500 [00:08<00:10, 25.57it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  45%|████▌     | 226/500 [00:08<00:11, 24.38it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  46%|████▌     | 229/500 [00:08<00:11, 24.37it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  46%|████▋     | 232/500 [00:08<00:10, 24.52it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  47%|████▋     | 235/500 [00:09<00:10, 25.09it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  48%|████▊     | 238/500 [00:09<00:10, 25.45it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  48%|████▊     | 241/500 [00:09<00:09, 25.98it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  49%|████▉     | 244/500 [00:09<00:09, 25.85it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  49%|████▉     | 247/500 [00:09<00:09, 25.89it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  50%|█████     | 250/500 [00:09<00:09, 26.07it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  51%|█████     | 253/500 [00:09<00:09, 26.33it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  51%|█████     | 256/500 [00:09<00:09, 26.41it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  52%|█████▏    | 259/500 [00:09<00:09, 26.16it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  52%|█████▏    | 262/500 [00:10<00:09, 26.33it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  53%|█████▎    | 265/500 [00:10<00:08, 26.18it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  54%|█████▎    | 268/500 [00:10<00:08, 26.20it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  54%|█████▍    | 271/500 [00:10<00:08, 25.72it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  55%|█████▍    | 274/500 [00:10<00:08, 25.83it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  55%|█████▌    | 277/500 [00:10<00:08, 25.78it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  56%|█████▌    | 280/500 [00:10<00:08, 26.16it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  57%|█████▋    | 283/500 [00:10<00:08, 26.24it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  57%|█████▋    | 286/500 [00:10<00:08, 26.13it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  58%|█████▊    | 289/500 [00:11<00:07, 26.40it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  58%|█████▊    | 292/500 [00:11<00:07, 26.18it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  59%|█████▉    | 295/500 [00:11<00:07, 26.31it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  60%|█████▉    | 298/500 [00:11<00:07, 25.59it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  60%|██████    | 301/500 [00:11<00:07, 25.58it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  61%|██████    | 304/500 [00:11<00:07, 25.86it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  61%|██████▏   | 307/500 [00:11<00:07, 26.25it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  62%|██████▏   | 310/500 [00:11<00:07, 26.21it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  63%|██████▎   | 313/500 [00:12<00:07, 26.44it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  63%|██████▎   | 316/500 [00:12<00:07, 26.16it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  64%|██████▍   | 319/500 [00:12<00:06, 26.50it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  64%|██████▍   | 322/500 [00:12<00:06, 26.66it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  65%|██████▌   | 325/500 [00:12<00:06, 26.60it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  66%|██████▌   | 328/500 [00:12<00:06, 26.66it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  66%|██████▌   | 331/500 [00:12<00:06, 26.44it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  67%|██████▋   | 334/500 [00:12<00:06, 26.47it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  67%|██████▋   | 337/500 [00:12<00:06, 26.36it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  68%|██████▊   | 340/500 [00:13<00:06, 26.27it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  69%|██████▊   | 343/500 [00:13<00:05, 26.37it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  69%|██████▉   | 346/500 [00:13<00:05, 26.16it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  70%|██████▉   | 349/500 [00:13<00:05, 26.21it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  70%|███████   | 352/500 [00:13<00:05, 26.08it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  71%|███████   | 355/500 [00:13<00:05, 26.22it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  72%|███████▏  | 358/500 [00:13<00:05, 26.35it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  72%|███████▏  | 361/500 [00:13<00:05, 26.54it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  73%|███████▎  | 364/500 [00:13<00:05, 25.72it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  73%|███████▎  | 367/500 [00:14<00:05, 25.68it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  74%|███████▍  | 370/500 [00:14<00:05, 25.54it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  75%|███████▍  | 373/500 [00:14<00:05, 25.32it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  75%|███████▌  | 376/500 [00:14<00:04, 25.36it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  76%|███████▌  | 379/500 [00:14<00:04, 25.19it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  76%|███████▋  | 382/500 [00:14<00:04, 25.52it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  77%|███████▋  | 385/500 [00:14<00:04, 25.16it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  78%|███████▊  | 388/500 [00:14<00:04, 24.60it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  78%|███████▊  | 391/500 [00:15<00:04, 23.75it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  79%|███████▉  | 394/500 [00:15<00:04, 24.12it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  79%|███████▉  | 397/500 [00:15<00:04, 24.48it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  80%|████████  | 400/500 [00:15<00:04, 24.12it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  81%|████████  | 403/500 [00:15<00:03, 24.58it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  81%|████████  | 406/500 [00:15<00:03, 24.81it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  82%|████████▏ | 409/500 [00:15<00:03, 25.04it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  82%|████████▏ | 412/500 [00:15<00:03, 24.99it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  83%|████████▎ | 415/500 [00:16<00:03, 24.82it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  84%|████████▎ | 418/500 [00:16<00:03, 24.71it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  84%|████████▍ | 421/500 [00:16<00:03, 24.67it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  85%|████████▍ | 424/500 [00:16<00:03, 24.83it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  85%|████████▌ | 427/500 [00:16<00:03, 23.87it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  86%|████████▌ | 430/500 [00:16<00:02, 23.75it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  87%|████████▋ | 433/500 [00:16<00:02, 23.94it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  87%|████████▋ | 436/500 [00:16<00:02, 24.01it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  88%|████████▊ | 439/500 [00:17<00:02, 24.30it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  88%|████████▊ | 442/500 [00:17<00:02, 24.33it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  89%|████████▉ | 445/500 [00:17<00:02, 24.56it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  90%|████████▉ | 448/500 [00:17<00:02, 24.51it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  90%|█████████ | 451/500 [00:17<00:01, 24.57it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  91%|█████████ | 454/500 [00:17<00:01, 24.91it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  91%|█████████▏| 457/500 [00:17<00:01, 24.87it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  92%|█████████▏| 460/500 [00:17<00:01, 24.49it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  93%|█████████▎| 463/500 [00:17<00:01, 24.62it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  93%|█████████▎| 466/500 [00:18<00:01, 24.81it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  94%|█████████▍| 469/500 [00:18<00:01, 25.03it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  94%|█████████▍| 472/500 [00:18<00:01, 25.09it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  95%|█████████▌| 475/500 [00:18<00:01, 24.91it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  96%|█████████▌| 478/500 [00:18<00:00, 25.23it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  96%|█████████▌| 481/500 [00:18<00:00, 25.01it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  97%|█████████▋| 484/500 [00:18<00:00, 25.22it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  97%|█████████▋| 487/500 [00:18<00:00, 24.90it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  98%|█████████▊| 490/500 [00:19<00:00, 24.96it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  99%|█████████▊| 493/500 [00:19<00:00, 24.96it/s]\u001b[A\n",
      "Searching with note10 constraint (55):  99%|█████████▉| 496/500 [00:19<00:00, 25.14it/s]\u001b[A\n",
      "Searching with note10 constraint (55): 100%|██████████| 500/500 [00:19<00:00, 25.69it/s]\u001b[A\n",
      "\n",
      "Searching with note10 constraint (60):   0%|          | 0/500 [00:00<?, ?it/s]\u001b[A\n",
      "Searching with note10 constraint (60):   1%|          | 3/500 [00:00<00:16, 29.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 55.00 msIt achieves 84.52% predicted accuracy with 54.99 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E6.0, K3), None)\n",
      "((O32, E3.0, K7), Identity)\n",
      "((O32, E4.0, K5), Identity)\n",
      "((O32, E3.0, K5), Identity)\n",
      "(SE(O48, E6.0, K7), None)\n",
      "(SE(O48, E4.0, K5), Identity)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E4.0, K5), Identity)\n",
      "((O96, E3.0, K7), None)\n",
      "((O96, E3.0, K5), Identity)\n",
      "((O96, E3.0, K7), Identity)\n",
      "((O96, E6.0, K5), Identity)\n",
      "(SE(O136, E4.0, K3), None)\n",
      "(SE(O136, E6.0, K5), Identity)\n",
      "(SE(O136, E6.0, K3), Identity)\n",
      "(SE(O136, E4.0, K5), Identity)\n",
      "(SE(O192, E6.0, K5), None)\n",
      "(SE(O192, E6.0, K7), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E4.0, K7), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  19.496007204055786\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Searching with note10 constraint (60):   1%|          | 6/500 [00:00<00:17, 28.76it/s]\u001b[A\n",
      "Searching with note10 constraint (60):   2%|▏         | 9/500 [00:00<00:17, 27.63it/s]\u001b[A\n",
      "Searching with note10 constraint (60):   2%|▏         | 12/500 [00:00<00:17, 27.51it/s]\u001b[A\n",
      "Searching with note10 constraint (60):   3%|▎         | 15/500 [00:00<00:17, 27.55it/s]\u001b[A\n",
      "Searching with note10 constraint (60):   4%|▎         | 18/500 [00:00<00:17, 27.79it/s]\u001b[A\n",
      "Searching with note10 constraint (60):   4%|▍         | 21/500 [00:00<00:17, 27.67it/s]\u001b[A\n",
      "Searching with note10 constraint (60):   5%|▍         | 24/500 [00:00<00:17, 27.25it/s]\u001b[A\n",
      "Searching with note10 constraint (60):   5%|▌         | 27/500 [00:00<00:17, 26.47it/s]\u001b[A\n",
      "Searching with note10 constraint (60):   6%|▌         | 30/500 [00:01<00:17, 26.54it/s]\u001b[A\n",
      "Searching with note10 constraint (60):   7%|▋         | 33/500 [00:01<00:17, 26.68it/s]\u001b[A\n",
      "Searching with note10 constraint (60):   7%|▋         | 36/500 [00:01<00:17, 26.64it/s]\u001b[A\n",
      "Searching with note10 constraint (60):   8%|▊         | 39/500 [00:01<00:17, 26.83it/s]\u001b[A\n",
      "Searching with note10 constraint (60):   8%|▊         | 42/500 [00:01<00:17, 26.58it/s]\u001b[A\n",
      "Searching with note10 constraint (60):   9%|▉         | 45/500 [00:01<00:17, 26.63it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  10%|▉         | 48/500 [00:01<00:16, 26.78it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  10%|█         | 51/500 [00:01<00:16, 26.94it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  11%|█         | 54/500 [00:02<00:16, 26.60it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  11%|█▏        | 57/500 [00:02<00:17, 26.02it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  12%|█▏        | 60/500 [00:02<00:16, 26.28it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  13%|█▎        | 63/500 [00:02<00:16, 26.26it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  13%|█▎        | 66/500 [00:02<00:16, 26.53it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  14%|█▍        | 69/500 [00:02<00:16, 26.45it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  14%|█▍        | 72/500 [00:02<00:16, 26.31it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  15%|█▌        | 75/500 [00:02<00:15, 26.77it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  16%|█▌        | 78/500 [00:02<00:15, 27.01it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  16%|█▌        | 81/500 [00:03<00:15, 26.81it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  17%|█▋        | 84/500 [00:03<00:15, 27.11it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  17%|█▋        | 87/500 [00:03<00:15, 27.40it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  18%|█▊        | 90/500 [00:03<00:15, 27.17it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  19%|█▊        | 93/500 [00:03<00:14, 27.16it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  19%|█▉        | 96/500 [00:03<00:14, 27.24it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  20%|█▉        | 99/500 [00:03<00:14, 27.39it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  20%|██        | 102/500 [00:03<00:14, 27.76it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  21%|██        | 105/500 [00:03<00:14, 27.86it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  22%|██▏       | 108/500 [00:04<00:14, 27.86it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  22%|██▏       | 111/500 [00:04<00:14, 27.71it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  23%|██▎       | 114/500 [00:04<00:14, 27.49it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  23%|██▎       | 117/500 [00:04<00:13, 27.67it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  24%|██▍       | 120/500 [00:04<00:13, 27.77it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  25%|██▍       | 123/500 [00:04<00:13, 27.41it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  25%|██▌       | 126/500 [00:04<00:13, 27.58it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  26%|██▌       | 129/500 [00:04<00:13, 27.36it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  26%|██▋       | 132/500 [00:04<00:13, 27.58it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  27%|██▋       | 135/500 [00:04<00:13, 27.25it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  28%|██▊       | 138/500 [00:05<00:13, 27.01it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  28%|██▊       | 141/500 [00:05<00:13, 27.26it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  29%|██▉       | 144/500 [00:05<00:13, 27.14it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  29%|██▉       | 147/500 [00:05<00:12, 27.32it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  30%|███       | 150/500 [00:05<00:13, 26.44it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  31%|███       | 153/500 [00:05<00:13, 26.46it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  31%|███       | 156/500 [00:05<00:12, 26.83it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  32%|███▏      | 159/500 [00:05<00:12, 26.69it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  32%|███▏      | 162/500 [00:05<00:12, 26.68it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  33%|███▎      | 165/500 [00:06<00:12, 26.68it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  34%|███▎      | 168/500 [00:06<00:12, 26.27it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  34%|███▍      | 171/500 [00:06<00:12, 26.64it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  35%|███▍      | 174/500 [00:06<00:12, 26.54it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  35%|███▌      | 177/500 [00:06<00:12, 26.83it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  36%|███▌      | 180/500 [00:06<00:11, 26.90it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  37%|███▋      | 183/500 [00:06<00:11, 27.03it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  37%|███▋      | 186/500 [00:06<00:11, 27.11it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  38%|███▊      | 189/500 [00:07<00:11, 27.27it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  38%|███▊      | 192/500 [00:07<00:11, 26.88it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  39%|███▉      | 195/500 [00:07<00:11, 25.97it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  40%|███▉      | 198/500 [00:07<00:11, 26.14it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  40%|████      | 201/500 [00:07<00:11, 26.55it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  41%|████      | 204/500 [00:07<00:11, 26.63it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  41%|████▏     | 207/500 [00:07<00:11, 26.56it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  42%|████▏     | 210/500 [00:07<00:10, 26.60it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  43%|████▎     | 213/500 [00:07<00:10, 26.51it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  43%|████▎     | 216/500 [00:08<00:10, 26.91it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  44%|████▍     | 219/500 [00:08<00:10, 26.80it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  44%|████▍     | 222/500 [00:08<00:10, 27.02it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  45%|████▌     | 225/500 [00:08<00:10, 27.12it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  46%|████▌     | 228/500 [00:08<00:09, 27.30it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  46%|████▌     | 231/500 [00:08<00:09, 27.32it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  47%|████▋     | 234/500 [00:08<00:09, 27.23it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  47%|████▋     | 237/500 [00:08<00:09, 27.45it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  48%|████▊     | 240/500 [00:08<00:09, 27.27it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  49%|████▊     | 243/500 [00:09<00:09, 27.20it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  49%|████▉     | 246/500 [00:09<00:09, 27.29it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  50%|████▉     | 249/500 [00:09<00:09, 27.49it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  50%|█████     | 252/500 [00:09<00:09, 26.99it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  51%|█████     | 255/500 [00:09<00:09, 27.13it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  52%|█████▏    | 258/500 [00:09<00:08, 27.13it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  52%|█████▏    | 261/500 [00:09<00:09, 26.52it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  53%|█████▎    | 264/500 [00:09<00:08, 26.51it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  53%|█████▎    | 267/500 [00:09<00:08, 26.66it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  54%|█████▍    | 270/500 [00:10<00:08, 26.49it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  55%|█████▍    | 273/500 [00:10<00:08, 26.42it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  55%|█████▌    | 276/500 [00:10<00:08, 26.78it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  56%|█████▌    | 279/500 [00:10<00:08, 27.04it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  56%|█████▋    | 282/500 [00:10<00:07, 27.40it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  57%|█████▋    | 285/500 [00:10<00:07, 27.22it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  58%|█████▊    | 288/500 [00:10<00:07, 27.42it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  58%|█████▊    | 291/500 [00:10<00:07, 27.58it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  59%|█████▉    | 294/500 [00:10<00:07, 27.65it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  59%|█████▉    | 297/500 [00:11<00:07, 27.69it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  60%|██████    | 300/500 [00:11<00:07, 27.80it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  61%|██████    | 303/500 [00:11<00:07, 27.66it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  61%|██████    | 306/500 [00:11<00:07, 27.60it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  62%|██████▏   | 309/500 [00:11<00:06, 27.81it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  62%|██████▏   | 312/500 [00:11<00:06, 27.72it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  63%|██████▎   | 315/500 [00:11<00:06, 27.71it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  64%|██████▎   | 318/500 [00:11<00:06, 27.80it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  64%|██████▍   | 321/500 [00:11<00:06, 27.89it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  65%|██████▍   | 324/500 [00:11<00:06, 27.78it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  65%|██████▌   | 327/500 [00:12<00:06, 27.34it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  66%|██████▌   | 330/500 [00:12<00:06, 26.61it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  67%|██████▋   | 333/500 [00:12<00:06, 26.30it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  67%|██████▋   | 336/500 [00:12<00:06, 26.42it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  68%|██████▊   | 339/500 [00:12<00:06, 26.64it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  68%|██████▊   | 342/500 [00:12<00:05, 27.11it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  69%|██████▉   | 345/500 [00:12<00:05, 27.52it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  70%|██████▉   | 348/500 [00:12<00:05, 27.14it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  70%|███████   | 351/500 [00:12<00:05, 27.06it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  71%|███████   | 354/500 [00:13<00:05, 27.00it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  71%|███████▏  | 357/500 [00:13<00:05, 27.33it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  72%|███████▏  | 360/500 [00:13<00:05, 27.52it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  73%|███████▎  | 363/500 [00:13<00:04, 27.46it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  73%|███████▎  | 366/500 [00:13<00:04, 27.10it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  74%|███████▍  | 369/500 [00:13<00:04, 27.34it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  74%|███████▍  | 372/500 [00:13<00:04, 25.66it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  75%|███████▌  | 375/500 [00:13<00:05, 24.94it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  76%|███████▌  | 378/500 [00:14<00:04, 25.24it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  76%|███████▌  | 381/500 [00:14<00:04, 25.72it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  77%|███████▋  | 384/500 [00:14<00:04, 26.17it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  77%|███████▋  | 387/500 [00:14<00:04, 26.37it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  78%|███████▊  | 390/500 [00:14<00:04, 26.59it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  79%|███████▊  | 393/500 [00:14<00:03, 26.91it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  79%|███████▉  | 396/500 [00:14<00:03, 27.43it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  80%|███████▉  | 399/500 [00:14<00:03, 27.45it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  80%|████████  | 402/500 [00:14<00:03, 27.55it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  81%|████████  | 405/500 [00:14<00:03, 27.67it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  82%|████████▏ | 408/500 [00:15<00:03, 27.63it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  82%|████████▏ | 411/500 [00:15<00:03, 27.37it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  83%|████████▎ | 414/500 [00:15<00:03, 27.31it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  83%|████████▎ | 417/500 [00:15<00:03, 27.09it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  84%|████████▍ | 420/500 [00:15<00:02, 27.10it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  85%|████████▍ | 423/500 [00:15<00:02, 26.92it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  85%|████████▌ | 426/500 [00:15<00:02, 26.66it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  86%|████████▌ | 429/500 [00:15<00:02, 26.75it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  86%|████████▋ | 432/500 [00:16<00:02, 26.40it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  87%|████████▋ | 435/500 [00:16<00:02, 26.75it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  88%|████████▊ | 438/500 [00:16<00:02, 26.92it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  88%|████████▊ | 441/500 [00:16<00:02, 27.03it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  89%|████████▉ | 444/500 [00:16<00:02, 27.26it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  89%|████████▉ | 447/500 [00:16<00:01, 27.28it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  90%|█████████ | 450/500 [00:16<00:01, 27.19it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  91%|█████████ | 453/500 [00:16<00:01, 27.06it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  91%|█████████ | 456/500 [00:16<00:01, 27.33it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  92%|█████████▏| 459/500 [00:16<00:01, 27.51it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  92%|█████████▏| 462/500 [00:17<00:01, 27.33it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  93%|█████████▎| 465/500 [00:17<00:01, 25.49it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  94%|█████████▎| 468/500 [00:17<00:01, 24.90it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  94%|█████████▍| 471/500 [00:17<00:01, 24.96it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  95%|█████████▍| 474/500 [00:17<00:01, 25.02it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  95%|█████████▌| 477/500 [00:17<00:00, 25.28it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  96%|█████████▌| 480/500 [00:17<00:00, 25.11it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  97%|█████████▋| 483/500 [00:17<00:00, 25.58it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  97%|█████████▋| 486/500 [00:18<00:00, 25.46it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  98%|█████████▊| 489/500 [00:18<00:00, 25.72it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  98%|█████████▊| 492/500 [00:18<00:00, 26.02it/s]\u001b[A\n",
      "Searching with note10 constraint (60):  99%|█████████▉| 495/500 [00:18<00:00, 25.89it/s]\u001b[A\n",
      "Searching with note10 constraint (60): 100%|██████████| 500/500 [00:18<00:00, 26.86it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found best architecture on note10 with latency <= 60.00 msIt achieves 84.74% predicted accuracy with 55.76 ms latency on note10.\n",
      "Architecture of the searched sub-net:\n",
      "3x3_Conv_O24_H_SWISH_BN\n",
      "(3x3_MBConv1_RELU_O24_BN, Identity)\n",
      "((O32, E6.0, K5), None)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E3.0, K3), Identity)\n",
      "((O32, E6.0, K3), Identity)\n",
      "(SE(O48, E6.0, K5), None)\n",
      "(SE(O48, E6.0, K7), Identity)\n",
      "(SE(O48, E4.0, K3), Identity)\n",
      "(SE(O48, E6.0, K3), Identity)\n",
      "((O96, E4.0, K5), None)\n",
      "((O96, E4.0, K7), Identity)\n",
      "((O96, E4.0, K7), Identity)\n",
      "((O96, E6.0, K7), Identity)\n",
      "(SE(O136, E4.0, K7), None)\n",
      "(SE(O136, E4.0, K7), Identity)\n",
      "(SE(O136, E4.0, K7), Identity)\n",
      "(SE(O136, E6.0, K5), Identity)\n",
      "(SE(O192, E6.0, K3), None)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E6.0, K5), Identity)\n",
      "(SE(O192, E3.0, K5), Identity)\n",
      "1x1_Conv_O1152_H_SWISH_BN\n",
      "1x1_Conv_O1536_H_SWISH\n",
      "1536x1000_Linear\n",
      "\n",
      "Time taken for latency constraint:  18.64280915260315\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(latency_constraints)):\n",
    "    lc = latency_constraints[i:i+1]\n",
    "    times[lc] = run_bottom_up_evolutionary_search(lc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_values([31.404531002044678, 25.56589913368225, 22.32000708580017, 20.48673677444458, 21.243536949157715, 26.576290130615234, 20.8565890789032, 19.848477125167847, 19.496007204055786, 18.64280915260315])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "times.values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DEMO\n",
    "\n",
    "Run the cell below to see the bottom up strategy in action!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "latency_constraints = (15, 20, 25, 30, 35)\n",
    "run_bottom_up_evolutionary_search(latency_constraints)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}