From d4137b9fdbad0152c6ea2e30d88bd98a1bf3824f Mon Sep 17 00:00:00 2001
From: aptsunny <aptsunny@tongji.edu.cn>
Date: Fri, 14 Oct 2022 11:50:00 +0800
Subject: [PATCH 01/18] update doc

---
 mmrazor/structures/subnet/candidate.py | 55 ++++++++++++++++++--------
 1 file changed, 38 insertions(+), 17 deletions(-)

diff --git a/mmrazor/structures/subnet/candidate.py b/mmrazor/structures/subnet/candidate.py
index 689db3814..35a305a8e 100644
--- a/mmrazor/structures/subnet/candidate.py
+++ b/mmrazor/structures/subnet/candidate.py
@@ -4,8 +4,8 @@
 
 
 class Candidates(UserList):
-    """The data structure of sampled candidate. The format is Union[Dict[Any,
-    Dict], List[Dict[Any, Dict]]].
+    """The data structure of sampled candidate. The format is Union[Dict[str,
+    Dict], List[Dict[str, Dict]]].
 
     Examples:
         >>> candidates = Candidates()
@@ -34,10 +34,11 @@ class Candidates(UserList):
         >>> candidates.scores
         [100.0, 0.0]
     """
-    _format_return = Union[Dict[Any, Dict], List[Dict[Any, Dict]]]
+    _format_return = Union[Dict[str, Dict], List[Dict[str, Dict]]]
+    _format_input = Union[Dict, Dict[str, Dict], List[Dict[str, Dict]]]
     _indicators = ('score', 'flops', 'params', 'latency')
 
-    def __init__(self, initdata: Optional[Any] = None):
+    def __init__(self, initdata: Optional[_format_input] = None):
         self.data = []
         if initdata is not None:
             initdata = self._format(initdata)
@@ -67,18 +68,33 @@ def subnets(self) -> List[Dict]:
         """The subnets of candidates."""
         return [eval(key) for item in self.data for key, _ in item.items()]
 
-    def _format(self, data: Any) -> _format_return:
-        """Transform [Dict, ...] to Union[Dict[Any, Dict], List[Dict[Any,
+    def _format(self, data: _format_input) -> _format_return:
+        """Transform [Dict, ...] to Union[Dict[str, Dict], List[Dict[str,
         Dict]]].
 
-        Three types of input are supported:
-            1. Dict.
-            2. Dict[Any, Dict].
-            3. List[Dict[Any, Dict]].
+        Args:
+            data: Three types of input are supported:
+                1. Dict: only include network information.
+                2. Dict[str, Dict]: network information and the corresponding
+                    resources.
+                3. List[Dict[str, Dict]]: multiple candidate information.
+
+        Returns:
+            Union[Dict[str, Dict], UserList[Dict[str, Dict]]]:
+                A dict that contains a pair of network information and the
+                corresponding Score | FLOPs | Params | Latency results in
+                each candidate.
+
+        Notes:
+            Score | FLOPs | Params | Latency:
+                1. a candidate resources with a default value of -1 indicates
+                    that it has not been estimated.
+                2. a candidate resources with a default value of 0 indicates
+                    that some indicators have been evaluated.
         """
 
-        def _format_item(cond: Any):
-            """Transform Dict to str(Dict)."""
+        def _format_item(cond: Dict) -> Dict[str, Dict]:
+            """Transform Dict to Dict[str, Dict]."""
             if len(cond.keys()) > 1 and isinstance(
                     list(cond.values())[0], str):
                 return {str(cond): {}.fromkeys(self._indicators, -1)}
@@ -97,12 +113,12 @@ def _format_item(cond: Any):
         else:
             return _format_item(data)
 
-    def append(self, item: Any) -> None:
+    def append(self, item: _format_input) -> None:
         """Append operation."""
         item = self._format(item)
         self.data.append(item)
 
-    def insert(self, i: int, item: Any) -> None:
+    def insert(self, i: int, item: _format_input) -> None:
         """Insert operation."""
         item = self._format(item)
         self.data.insert(i, item)
@@ -129,10 +145,15 @@ def set_resources(self,
         for _, value in self.data[i].items():
             value[key_indicator] = resources
 
-    def sort_by(self, key_indicator='score', reverse=True):
-        """Sort by a specific indicator.
+    def sort_by(self,
+                key_indicator: str = 'score',
+                reverse: bool = True) -> None:
+        """Sort by a specific indicator in descending order.
 
-        Default score.
+        Args:
+            key_indicator (str): sort all candidates by key_indicator.
+                Defaults to 'score'.
+            reverse (bool): sort all candidates in descending order.
         """
         self.data.sort(
             key=lambda x: list(x.values())[0][key_indicator], reverse=reverse)

From 589aec050f57d1877fa8d53e8eb5753977ca6e65 Mon Sep 17 00:00:00 2001
From: aptsunny <aptsunny@tongji.edu.cn>
Date: Fri, 14 Oct 2022 15:25:52 +0800
Subject: [PATCH 02/18] add support type

---
 .../engine/runner/evolution_search_loop.py    | 116 +++++++++++++-----
 mmrazor/engine/runner/subnet_sampler_loop.py  |  52 ++++----
 mmrazor/engine/runner/utils/__init__.py       |   4 +-
 mmrazor/engine/runner/utils/check.py          |  33 +++--
 mmrazor/structures/subnet/candidate.py        |  17 +--
 .../test_models/test_subnet/test_candidate.py |  80 ++++++++++--
 .../test_evolution_search_loop.py             |  54 +++++---
 .../test_runners/test_subnet_sampler_loop.py  |   9 +-
 tests/test_runners/test_utils/test_check.py   |  38 +++---
 9 files changed, 268 insertions(+), 135 deletions(-)

diff --git a/mmrazor/engine/runner/evolution_search_loop.py b/mmrazor/engine/runner/evolution_search_loop.py
index a9a76b383..692c0fc94 100644
--- a/mmrazor/engine/runner/evolution_search_loop.py
+++ b/mmrazor/engine/runner/evolution_search_loop.py
@@ -3,7 +3,7 @@
 import os.path as osp
 import random
 import warnings
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Union
 
 import torch
 from mmengine import fileio
@@ -17,7 +17,7 @@
 from mmrazor.registry import LOOPS
 from mmrazor.structures import Candidates, export_fix_subnet
 from mmrazor.utils import SupportRandomSubnet
-from .utils import check_subnet_flops, crossover
+from .utils import check_subnet_resources, crossover
 
 
 @LOOPS.register_module()
@@ -41,10 +41,10 @@ class EvolutionSearchLoop(EpochBasedTrainLoop):
         num_crossover (int): The number of candidates got by crossover.
             Defaults to 25.
         mutate_prob (float): The probability of mutation. Defaults to 0.1.
-        flops_range (tuple, optional): It is used for screening candidates.
-        resource_estimator_cfg (dict): The config for building estimator, which
-            is be used to estimate the flops of sampled subnet. Defaults to
-            None, which means default config is used.
+        constraints_range (Dict[str, Any]): Constraints to be used for
+            screening candidates. Defaults to dict(flops=(0, 330)).
+        resource_estimator_cfg (Dict[str, Any]): Used for building a
+            resource estimator. Defaults to dict().
         score_key (str): Specify one metric in evaluation results to score
             candidates. Defaults to 'accuracy_top-1'.
         init_candidates (str, optional): The candidates file path, which is
@@ -64,8 +64,8 @@ def __init__(self,
                  num_mutation: int = 25,
                  num_crossover: int = 25,
                  mutate_prob: float = 0.1,
-                 flops_range: Optional[Tuple[float, float]] = (0., 330.),
-                 resource_estimator_cfg: Optional[dict] = None,
+                 constraints_range: Dict[str, Any] = dict(flops=(0., 330.)),
+                 resource_estimator_cfg: Dict[str, Any] = dict(),
                  score_key: str = 'accuracy/top1',
                  init_candidates: Optional[str] = None) -> None:
         super().__init__(runner, dataloader, max_epochs)
@@ -83,7 +83,9 @@ def __init__(self,
 
         self.num_candidates = num_candidates
         self.top_k = top_k
-        self.flops_range = flops_range
+        self.constraints_range = constraints_range
+        self.key_indicator = list(self.constraints_range.keys())[0]
+        self.estimator_cfg = resource_estimator_cfg
         self.score_key = score_key
         self.num_mutation = num_mutation
         self.num_crossover = num_crossover
@@ -99,10 +101,7 @@ def __init__(self,
                 correct init candidates file'
 
         self.top_k_candidates = Candidates()
-        if resource_estimator_cfg is None:
-            self.estimator = ResourceEstimator()
-        else:
-            self.estimator = ResourceEstimator(**resource_estimator_cfg)
+        self.estimator = ResourceEstimator(**resource_estimator_cfg)
 
         if self.runner.distributed:
             self.model = runner.model.module
@@ -144,33 +143,50 @@ def run_epoch(self) -> None:
                                 f'{scores_before}')
 
         self.candidates.extend(self.top_k_candidates)
-        self.candidates.sort(key=lambda x: x[1], reverse=True)
-        self.top_k_candidates = Candidates(self.candidates[:self.top_k])
+        self.candidates.sort_by(key_indicator='score', reverse=True)
+        self.top_k_candidates = Candidates(self.candidates.data[:self.top_k])
 
         scores_after = self.top_k_candidates.scores
         self.runner.logger.info(f'top k scores after update: '
                                 f'{scores_after}')
 
+        self.candidates_mutator_crossover = Candidates()
         mutation_candidates = self.gen_mutation_candidates()
+        self.candidates_mutator_crossover.extend(mutation_candidates)
         crossover_candidates = self.gen_crossover_candidates()
-        candidates = mutation_candidates + crossover_candidates
-        assert len(candidates) <= self.num_candidates, 'Total of mutation and \
+        self.candidates_mutator_crossover.extend(crossover_candidates)
+
+        assert len(self.candidates_mutator_crossover
+                   ) <= self.num_candidates, 'Total of mutation and \
             crossover should be no more than the number of candidates.'
 
-        self.candidates = Candidates(candidates)
+        self.candidates = self.candidates_mutator_crossover
         self._epoch += 1
 
     def sample_candidates(self) -> None:
         """Update candidate pool contains specified number of candicates."""
+        candidates_resources = []
+        init_candidates = len(self.candidates)
         if self.runner.rank == 0:
             while len(self.candidates) < self.num_candidates:
+                is_pass = False
                 candidate = self.model.sample_subnet()
-                if self._check_constraints(random_subnet=candidate):
+                is_pass, result = self._check_constraints(
+                    random_subnet=candidate, need_feedback=True)
+                if is_pass:
                     self.candidates.append(candidate)
+                    candidates_resources.append(result)
+            self.candidates = Candidates(self.candidates.data)
         else:
-            self.candidates = Candidates([None] * self.num_candidates)
+            self.candidates = Candidates([dict()] * self.num_candidates)
         # broadcast candidates to val with multi-GPUs.
         broadcast_object_list(self.candidates.data)
+        assert init_candidates + len(
+            candidates_resources) == self.num_candidates
+        for i in range(len(candidates_resources)):
+            resources = candidates_resources[i][self.key_indicator] \
+                if len(candidates_resources[i]) != 0 else 0.
+            self.candidates.set_resources(i + init_candidates, resources)
 
     def update_candidates_scores(self) -> None:
         """Validate candicate one by one from the candicate pool, and update
@@ -182,12 +198,15 @@ def update_candidates_scores(self) -> None:
                 if len(metrics) != 0 else 0.
             self.candidates.set_score(i, score)
             self.runner.logger.info(
-                f'Epoch:[{self._epoch}/{self._max_epochs}] '
+                f'Epoch:[{self.runner.epoch}/{self.max_epochs}] '
                 f'Candidate:[{i + 1}/{self.num_candidates}] '
+                f'{self.key_indicator}: '
+                f'{self.candidates.resources(self.key_indicator)[i]} '
                 f'Score:{score}')
 
-    def gen_mutation_candidates(self) -> List:
+    def gen_mutation_candidates(self):
         """Generate specified number of mutation candicates."""
+        mutation_resources = []
         mutation_candidates: List = []
         max_mutate_iters = self.num_mutation * 10
         mutate_iter = 0
@@ -198,12 +217,25 @@ def gen_mutation_candidates(self) -> List:
 
             mutation_candidate = self._mutation()
 
-            if self._check_constraints(random_subnet=mutation_candidate):
+            is_pass = False
+            is_pass, result = self._check_constraints(
+                random_subnet=mutation_candidate, need_feedback=True)
+            if is_pass:
                 mutation_candidates.append(mutation_candidate)
+                mutation_resources.append(result)
+
+        mutation_candidates = Candidates(mutation_candidates)
+
+        for i in range(self.num_mutation):
+            resources = mutation_resources[i][self.key_indicator] \
+                if len(mutation_resources[i]) != 0 else 0.
+            mutation_candidates.set_resources(i, resources)
+
         return mutation_candidates
 
-    def gen_crossover_candidates(self) -> List:
+    def gen_crossover_candidates(self):
         """Generate specofied number of crossover candicates."""
+        crossover_resources = []
         crossover_candidates: List = []
         crossover_iter = 0
         max_crossover_iters = self.num_crossover * 10
@@ -214,8 +246,19 @@ def gen_crossover_candidates(self) -> List:
 
             crossover_candidate = self._crossover()
 
-            if self._check_constraints(random_subnet=crossover_candidate):
+            is_pass = False
+            is_pass, result = self._check_constraints(
+                random_subnet=crossover_candidate, need_feedback=True)
+            if is_pass:
                 crossover_candidates.append(crossover_candidate)
+                crossover_resources.append(result)
+        crossover_candidates = Candidates(crossover_candidates)
+
+        for i in range(self.num_crossover):
+            resources = crossover_resources[i][self.key_indicator] \
+                if len(crossover_resources[i]) != 0 else 0.
+            crossover_candidates.set_resources(i, resources)
+
         return crossover_candidates
 
     def _mutation(self) -> SupportRandomSubnet:
@@ -239,7 +282,7 @@ def _resume(self):
             for k in searcher_resume.keys():
                 setattr(self, k, searcher_resume[k])
             epoch_start = int(searcher_resume['_epoch'])
-            self._max_epochs = self._max_epochs - epoch_start
+            self._max_epochs = self.max_epochs - epoch_start
             self.runner.logger.info('#' * 100)
             self.runner.logger.info(f'Resume from epoch: {epoch_start}')
             self.runner.logger.info('#' * 100)
@@ -263,7 +306,7 @@ def _val_candidate(self) -> Dict:
         self.runner.model.eval()
         for data_batch in self.dataloader:
             outputs = self.runner.model.val_step(data_batch)
-            self.evaluator.process(data_samples=outputs, data_batch=data_batch)
+            self.evaluator.process(outputs, data_batch)
         metrics = self.evaluator.evaluate(len(self.dataloader.dataset))
         return metrics
 
@@ -283,11 +326,11 @@ def _save_searcher_ckpt(self) -> None:
                 osp.join(self.runner.work_dir,
                          f'search_epoch_{self._epoch}.pkl'))
             self.runner.logger.info(
-                f'Epoch:[{self._epoch}/{self._max_epochs}], top1_score: '
+                f'Epoch:[{self._epoch}/{self.max_epochs}], top1_score: '
                 f'{self.top_k_candidates.scores[0]}')
 
             if self.max_keep_ckpts > 0:
-                cur_ckpt = self._epoch + 1
+                cur_ckpt = self.runner.epoch + 1
                 redundant_ckpts = range(1, cur_ckpt - self.max_keep_ckpts)
                 for _step in redundant_ckpts:
                     ckpt_path = osp.join(self.runner.work_dir,
@@ -295,16 +338,21 @@ def _save_searcher_ckpt(self) -> None:
                     if osp.isfile(ckpt_path):
                         os.remove(ckpt_path)
 
-    def _check_constraints(self, random_subnet: SupportRandomSubnet) -> bool:
+    def _check_constraints(self,
+                           random_subnet: SupportRandomSubnet,
+                           need_feedback: bool = False):
         """Check whether is beyond constraints.
 
         Returns:
-            bool: The result of checking.
+            bool, result: The result of checking.
         """
-        is_pass = check_subnet_flops(
+        is_pass, results = check_subnet_resources(
             model=self.model,
             subnet=random_subnet,
             estimator=self.estimator,
-            flops_range=self.flops_range)
+            constraints_range=self.constraints_range)
 
-        return is_pass
+        if need_feedback:
+            return is_pass, results
+        else:
+            return is_pass
diff --git a/mmrazor/engine/runner/subnet_sampler_loop.py b/mmrazor/engine/runner/subnet_sampler_loop.py
index 1127aab21..cc89086da 100644
--- a/mmrazor/engine/runner/subnet_sampler_loop.py
+++ b/mmrazor/engine/runner/subnet_sampler_loop.py
@@ -3,7 +3,7 @@
 import os
 import random
 from abc import abstractmethod
-from typing import Dict, List, Optional, Sequence, Tuple, Union
+from typing import Any, Dict, List, Sequence, Tuple, Union
 
 import torch
 from mmengine import fileio
@@ -16,7 +16,7 @@
 from mmrazor.registry import LOOPS
 from mmrazor.structures import Candidates
 from mmrazor.utils import SupportRandomSubnet
-from .utils import check_subnet_flops
+from .utils import check_subnet_resources
 
 
 class BaseSamplerTrainLoop(IterBasedTrainLoop):
@@ -77,18 +77,15 @@ def run_iter(self, data_batch: Sequence[dict]) -> None:
 @LOOPS.register_module()
 class GreedySamplerTrainLoop(BaseSamplerTrainLoop):
     """IterBasedTrainLoop for greedy sampler.
-
     In GreedySamplerTrainLoop, `Greedy` means that only use some top
     sampled candidates to train the supernet. So GreedySamplerTrainLoop mainly
     picks the top candidates based on their val socres, then use them to train
     the supernet one by one.
-
     Steps:
         1. Sample from the supernet and the candidates.
         2. Validate these sampled candidates to get each candidate's score.
         3. Get top-k candidates based on their scores, then use them to train
         the supernet one by one.
-
     Args:
         runner (Runner): A reference of runner.
         dataloader (Dataloader or dict): A dataloader object or a dict to
@@ -102,10 +99,11 @@ class GreedySamplerTrainLoop(BaseSamplerTrainLoop):
         val_interval (int): Validation interval. Defaults to 1000.
         score_key (str): Specify one metric in evaluation results to score
             candidates. Defaults to 'accuracy_top-1'.
-        flops_range (dict): Constraints to be used for screening candidates.
+        constraints_range (Dict[str, Any]): Constraints to be used for
+            screening candidates. Defaults to dict(flops=(0, 330)).
         resource_estimator_cfg (dict): The config for building estimator, which
             is be used to estimate the flops of sampled subnet. Defaults to
-            None, which means default config is used.
+            None, which means default config is used. Defaults to dict().
         num_candidates (int): The number of the candidates consist of samples
             from supernet and itself. Defaults to 1000.
         num_samples (int): The number of sample in each sampling subnet.
@@ -139,8 +137,8 @@ def __init__(self,
                  val_begin: int = 1,
                  val_interval: int = 1000,
                  score_key: str = 'accuracy/top1',
-                 flops_range: Optional[Tuple[float, float]] = (0., 330),
-                 resource_estimator_cfg: Optional[dict] = None,
+                 constraints_range: Dict[str, Any] = dict(flops=(0, 330)),
+                 resource_estimator_cfg: Dict[str, Any] = dict(),
                  num_candidates: int = 1000,
                  num_samples: int = 10,
                  top_k: int = 5,
@@ -163,7 +161,7 @@ def __init__(self,
             self.evaluator = evaluator
 
         self.score_key = score_key
-        self.flops_range = flops_range
+        self.constraints_range = constraints_range
         self.num_candidates = num_candidates
         self.num_samples = num_samples
         self.top_k = top_k
@@ -177,10 +175,7 @@ def __init__(self,
 
         self.candidates = Candidates()
         self.top_k_candidates = Candidates()
-        if resource_estimator_cfg is None:
-            self.estimator = ResourceEstimator()
-        else:
-            self.estimator = ResourceEstimator(**resource_estimator_cfg)
+        self.estimator = ResourceEstimator(**resource_estimator_cfg)
 
     def run(self) -> None:
         """Launch training."""
@@ -230,9 +225,11 @@ def sample_subnet(self) -> SupportRandomSubnet:
 
             self.update_candidates_scores()
 
-            self.candidates.sort(key=lambda x: x[1], reverse=True)
-            self.candidates = Candidates(self.candidates[:self.num_candidates])
-            self.top_k_candidates = Candidates(self.candidates[:self.top_k])
+            self.candidates.sort_by(key_indicator='score', reverse=True)
+            self.candidates = Candidates(
+                self.candidates.data[:self.num_candidates])
+            self.top_k_candidates = Candidates(
+                self.candidates.data[:self.top_k])
 
             top1_score = self.top_k_candidates.scores[0]
             if (self._iter % self.val_interval) < self.top_k:
@@ -243,7 +240,7 @@ def sample_subnet(self) -> SupportRandomSubnet:
                     f'{num_sample_from_supernet}/{self.num_samples} '
                     f'top1_score {top1_score:.3f} '
                     f'cur_num_candidates: {len(self.candidates)}')
-        return self.top_k_candidates.pop(0)[0]
+        return self.top_k_candidates.subnets[0]
 
     def update_cur_prob(self, cur_iter: int) -> None:
         """update current probablity of sampling from the candidates, which is
@@ -278,7 +275,7 @@ def get_candidates_with_sample(self,
         for _ in range(num_samples):
             if random.random() >= self.cur_prob or len(self.candidates) == 0:
                 subnet = self._sample_from_supernet()
-                if self._check_constraints(subnet):
+                if self._check_constraints(subnet, need_feedback=False):
                     sampled_candidates.append(subnet)
                 num_sample_from_supernet += 1
             else:
@@ -312,22 +309,27 @@ def _sample_from_supernet(self) -> SupportRandomSubnet:
     def _sample_from_candidates(self) -> SupportRandomSubnet:
         """Sample from the candidates."""
         assert len(self.candidates) > 0
-        subnet = random.choice(self.candidates)
+        subnet = random.choice(self.candidates.data)
         return subnet
 
-    def _check_constraints(self, random_subnet: SupportRandomSubnet) -> bool:
+    def _check_constraints(self,
+                           random_subnet: SupportRandomSubnet,
+                           need_feedback: bool = False):
         """Check whether is beyond constraints.
 
         Returns:
-            bool: The result of checking.
+            bool, result: The result of checking.
         """
-        is_pass = check_subnet_flops(
+        is_pass, results = check_subnet_resources(
             model=self.model,
             subnet=random_subnet,
             estimator=self.estimator,
-            flops_range=self.flops_range)
+            constraints_range=self.constraints_range)
 
-        return is_pass
+        if need_feedback:
+            return is_pass, results
+        else:
+            return is_pass
 
     def _save_candidates(self) -> None:
         """Save the candidates to init the next searching."""
diff --git a/mmrazor/engine/runner/utils/__init__.py b/mmrazor/engine/runner/utils/__init__.py
index ec2f2cb29..557002e2c 100644
--- a/mmrazor/engine/runner/utils/__init__.py
+++ b/mmrazor/engine/runner/utils/__init__.py
@@ -1,5 +1,5 @@
 # Copyright (c) OpenMMLab. All rights reserved.
-from .check import check_subnet_flops
+from .check import check_subnet_resources
 from .genetic import crossover
 
-__all__ = ['crossover', 'check_subnet_flops']
+__all__ = ['crossover', 'check_subnet_resources']
diff --git a/mmrazor/engine/runner/utils/check.py b/mmrazor/engine/runner/utils/check.py
index e2fdcfcc6..d50ddc74b 100644
--- a/mmrazor/engine/runner/utils/check.py
+++ b/mmrazor/engine/runner/utils/check.py
@@ -1,8 +1,6 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import copy
-from typing import Optional, Tuple
-
-import torch.nn as nn
+from typing import Any, Dict
 
 from mmrazor.models import ResourceEstimator
 from mmrazor.structures import export_fix_subnet, load_fix_subnet
@@ -15,18 +13,19 @@
     BaseDetector = get_placeholder('mmdet')
 
 
-def check_subnet_flops(
-        model: nn.Module,
-        subnet: SupportRandomSubnet,
-        estimator: ResourceEstimator,
-        flops_range: Optional[Tuple[float, float]] = None) -> bool:
-    """Check whether is beyond flops constraints.
+def check_subnet_resources(model,
+                           subnet: SupportRandomSubnet,
+                           estimator: ResourceEstimator,
+                           constraints_range: Dict[str,
+                                                   Any] = dict(flops=(0,
+                                                                      330))):
+    """Check whether is beyond resources constraints.
 
     Returns:
         bool: The result of checking.
     """
-    if flops_range is None:
-        return True
+    if constraints_range is None:
+        return True, None
 
     assert hasattr(model, 'set_subnet') and hasattr(model, 'architecture')
     model.set_subnet(subnet)
@@ -40,9 +39,9 @@ def check_subnet_flops(
     else:
         results = estimator.estimate(model=model_to_check)
 
-    flops = results['flops']
-    flops_mix, flops_max = flops_range
-    if flops_mix <= flops <= flops_max:  # type: ignore
-        return True
-    else:
-        return False
+    for k, v in constraints_range.items():
+        if not isinstance(v, (list, tuple)):
+            v = (0, v)
+        if results[k] < v[0] or results[k] > v[1]:
+            return False, results
+    return True, results
diff --git a/mmrazor/structures/subnet/candidate.py b/mmrazor/structures/subnet/candidate.py
index cea0fc534..44a1ad264 100644
--- a/mmrazor/structures/subnet/candidate.py
+++ b/mmrazor/structures/subnet/candidate.py
@@ -34,7 +34,8 @@ class Candidates(UserList):
         [100.0, 0.0]
     """
     _format_return = Union[Dict[str, Dict], List[Dict[str, Dict]]]
-    _format_input = Union[Dict, Dict[str, Dict], List[Dict[str, Dict]]]
+    _format_input = Union[Dict, List[Dict], Dict[str, Dict], List[Dict[str,
+                                                                       Dict]]]
     _indicators = ('score', 'flops', 'params', 'latency')
 
     def __init__(self, initdata: Optional[_format_input] = None):
@@ -72,16 +73,18 @@ def _format(self, data: _format_input) -> _format_return:
         Dict]]].
 
         Args:
-            data: Three types of input are supported:
+            data: Four types of input are supported:
                 1. Dict: only include network information.
-                2. Dict[str, Dict]: network information and the corresponding
+                2. List[Dict]: multiple candidates only include network
+                    information.
+                3. Dict[str, Dict]: network information and the corresponding
                     resources.
-                3. List[Dict[str, Dict]]: multiple candidate information.
+                4. List[Dict[str, Dict]]: multiple candidate information.
         Returns:
             Union[Dict[str, Dict], UserList[Dict[str, Dict]]]:
-                A dict that contains a pair of network information and the
-                corresponding Score | FLOPs | Params | Latency results in
-                each candidate.
+                A dict or a list of dict that contains a pair of network
+                information and the corresponding Score | FLOPs | Params |
+                Latency results in each candidate.
         Notes:
             Score | FLOPs | Params | Latency:
                 1. a candidate resources with a default value of -1 indicates
diff --git a/tests/test_models/test_subnet/test_candidate.py b/tests/test_models/test_subnet/test_candidate.py
index 4cf44846d..fda3a3a02 100644
--- a/tests/test_models/test_subnet/test_candidate.py
+++ b/tests/test_models/test_subnet/test_candidate.py
@@ -10,7 +10,22 @@ class TestCandidates(TestCase):
 
     def setUp(self) -> None:
         self.fake_subnet = {'1': 'choice1', '2': 'choice2'}
-        self.fake_subnet_with_score = (self.fake_subnet, 1.)
+        self.fake_subnet_with_resource = {
+            str(self.fake_subnet): {
+                'score': 0.,
+                'flops': 50.,
+                'params': 0.,
+                'latency': 0.
+            }
+        }
+        self.fake_subnet_with_score = {
+            str(self.fake_subnet): {
+                'score': 99.,
+                'flops': 0.,
+                'params': 0.,
+                'latency': 0.
+            }
+        }
 
     def test_init(self):
         # initlist is None
@@ -28,11 +43,16 @@ def test_scores(self):
         # test property: scores
         data = [self.fake_subnet_with_score] * 2
         candidates = Candidates(data)
-        self.assertEqual(candidates.scores, [1., 1.])
+        self.assertEqual(candidates.scores, [99., 99.])
+
+    def test_resources(self):
+        data = [self.fake_subnet_with_resource] * 2
+        candidates = Candidates(data)
+        self.assertEqual(candidates.resources('flops'), [50., 50.])
 
     def test_subnets(self):
         # test property: subnets
-        data = [self.fake_subnet_with_score] * 2
+        data = [self.fake_subnet] * 2
         candidates = Candidates(data)
         self.assertEqual(candidates.subnets, [self.fake_subnet] * 2)
 
@@ -41,17 +61,20 @@ def test_append(self):
         candidates = Candidates()
         candidates.append(self.fake_subnet)
         self.assertEqual(len(candidates), 1)
-        # item is tuple
+        # item is List
         candidates = Candidates()
-        candidates.append(self.fake_subnet_with_score)
-        self.assertEqual(len(candidates), 1)
+        candidates.append([self.fake_subnet_with_score])
+        # item is Candidates
+        candidates_2 = Candidates([self.fake_subnet_with_resource])
+        candidates.append(candidates_2)
+        self.assertEqual(len(candidates), 2)
 
     def test_insert(self):
         # item is dict
-        candidates = Candidates([self.fake_subnet_with_score])
+        candidates = Candidates(self.fake_subnet_with_score)
         candidates.insert(1, self.fake_subnet)
         self.assertEqual(len(candidates), 2)
-        # item is tuple
+        # item is List
         candidates = Candidates([self.fake_subnet_with_score])
         candidates.insert(1, self.fake_subnet_with_score)
         self.assertEqual(len(candidates), 2)
@@ -61,13 +84,46 @@ def test_extend(self):
         candidates = Candidates([self.fake_subnet_with_score])
         candidates.extend([self.fake_subnet])
         self.assertEqual(len(candidates), 2)
-        # other is UserList
+        # other is Candidates
         candidates = Candidates([self.fake_subnet_with_score])
-        candidates.extend(UserList([self.fake_subnet_with_score]))
+        candidates_2 = Candidates([self.fake_subnet_with_resource])
+        candidates.extend(candidates_2)
         self.assertEqual(len(candidates), 2)
 
+    def test_set_resources(self):
+        # test set_resources
+        candidates = Candidates([self.fake_subnet])
+        for kk in ['flops', 'params', 'latency']:
+            candidates.set_resources(0, 49.9, kk)
+            self.assertEqual(candidates.resources(kk)[0], 49.9)
+        candidates.insert(0, self.fake_subnet_with_resource)
+        self.assertEqual(len(candidates), 2)
+        self.assertEqual(candidates.resources('flops'), [50., 49.9])
+        self.assertEqual(candidates.resources('latency'), [0., 49.9])
+
     def test_set_score(self):
         # test set_score
         candidates = Candidates([self.fake_subnet_with_score])
-        candidates.set_score(0, 0.5)
-        self.assertEqual(candidates[0][1], 0.5)
+        candidates.set_score(0, 100.0)
+        self.assertEqual(candidates.scores[0], 100.)
+
+        candidates = Candidates([self.fake_subnet_with_score])
+        candidates.set_score(0, 100.0)
+        candidates.extend(UserList([self.fake_subnet_with_resource]))
+        candidates.set_score(1, 99.9)
+        self.assertEqual(candidates.scores, [100., 99.9])
+
+    def test_sort(self):
+        # test set_score
+        candidates = Candidates([self.fake_subnet_with_score])
+        candidates.extend(UserList([self.fake_subnet_with_resource]))
+        candidates.insert(0, self.fake_subnet)
+        candidates.set_score(0, 100.)
+        candidates.set_score(2, 98.)
+        self.assertEqual(candidates.scores, [100., 99., 98.])
+        candidates.sort_by(key_indicator='score', reverse=False)
+        self.assertEqual(candidates.scores, [98., 99., 100.])
+        candidates.sort_by(key_indicator='latency')
+        self.assertEqual(candidates.scores, [98., 99., 100.])
+        candidates.sort_by(key_indicator='flops', reverse=False)
+        self.assertEqual(candidates.scores, [100., 99., 98.])
diff --git a/tests/test_runners/test_evolution_search_loop.py b/tests/test_runners/test_evolution_search_loop.py
index f30019274..bb335d4f6 100644
--- a/tests/test_runners/test_evolution_search_loop.py
+++ b/tests/test_runners/test_evolution_search_loop.py
@@ -82,7 +82,7 @@ def setUp(self):
             num_mutation=2,
             num_crossover=2,
             mutate_prob=0.1,
-            flops_range=None,
+            constraints_range=dict(flops=(0, 330)),
             score_key='coco/bbox_mAP')
         self.train_cfg = Config(train_cfg)
         self.runner = MagicMock(spec=ToyRunner)
@@ -103,7 +103,7 @@ def test_init(self):
 
         # test init_candidates is not None
         fake_subnet = {'1': 'choice1', '2': 'choice2'}
-        fake_candidates = Candidates((fake_subnet, 0.))
+        fake_candidates = Candidates(fake_subnet)
         init_candidates_path = os.path.join(self.temp_dir, 'candidates.yaml')
         fileio.dump(fake_candidates, init_candidates_path)
         loop_cfg.init_candidates = init_candidates_path
@@ -111,29 +111,34 @@ def test_init(self):
         self.assertIsInstance(loop, EvolutionSearchLoop)
         self.assertEqual(loop.candidates, fake_candidates)
 
-    @patch('mmrazor.engine.runner.evolution_search_loop.export_fix_subnet')
-    def test_run_epoch(self, mock_export_fix_subnet):
+    @patch('mmrazor.engine.runner.utils.check.load_fix_subnet')
+    @patch('mmrazor.engine.runner.utils.check.export_fix_subnet')
+    @patch('mmrazor.models.task_modules.estimators.resource_estimator.'
+           'get_model_flops_params')
+    def test_run_epoch(self, flops_params, mock_export_fix_subnet,
+                       load_status):
         # test_run_epoch: distributed == False
         loop_cfg = copy.deepcopy(self.train_cfg)
         loop_cfg.runner = self.runner
         loop_cfg.dataloader = self.dataloader
         loop_cfg.evaluator = self.evaluator
+        loop_cfg.runner.epoch = 1
         loop = LOOPS.build(loop_cfg)
         self.runner.rank = 0
-        loop._epoch = 1
         self.runner.distributed = False
         self.runner.work_dir = self.temp_dir
         fake_subnet = {'1': 'choice1', '2': 'choice2'}
-        self.runner.model.sample_subnet = MagicMock(return_value=fake_subnet)
+        loop.model.sample_subnet = MagicMock(return_value=fake_subnet)
+        load_status.return_value = True
+        flops_params.return_value = 0, 0
         loop.run_epoch()
         self.assertEqual(len(loop.candidates), 4)
         self.assertEqual(len(loop.top_k_candidates), 2)
-        self.assertEqual(loop._epoch, 2)
+        self.assertEqual(loop._epoch, 1)
 
         # test_run_epoch: distributed == True
         loop = LOOPS.build(loop_cfg)
         self.runner.rank = 0
-        loop._epoch = 1
         self.runner.distributed = True
         self.runner.work_dir = self.temp_dir
         fake_subnet = {'1': 'choice1', '2': 'choice2'}
@@ -141,26 +146,27 @@ def test_run_epoch(self, mock_export_fix_subnet):
         loop.run_epoch()
         self.assertEqual(len(loop.candidates), 4)
         self.assertEqual(len(loop.top_k_candidates), 2)
-        self.assertEqual(loop._epoch, 2)
+        self.assertEqual(loop._epoch, 1)
 
         # test_check_constraints
-        loop_cfg.flops_range = (0, 100)
+        loop_cfg.constraints_range = dict(params=(0, 100))
         loop = LOOPS.build(loop_cfg)
         self.runner.rank = 0
-        loop._epoch = 1
         self.runner.distributed = True
         self.runner.work_dir = self.temp_dir
         fake_subnet = {'1': 'choice1', '2': 'choice2'}
         loop.model.sample_subnet = MagicMock(return_value=fake_subnet)
-        loop._check_constraints = MagicMock(return_value=True)
+        flops_params.return_value = (50., 1)
         mock_export_fix_subnet.return_value = fake_subnet
         loop.run_epoch()
         self.assertEqual(len(loop.candidates), 4)
         self.assertEqual(len(loop.top_k_candidates), 2)
-        self.assertEqual(loop._epoch, 2)
+        self.assertEqual(loop._epoch, 1)
 
-    @patch('mmrazor.engine.runner.evolution_search_loop.export_fix_subnet')
-    def test_run(self, mock_export_fix_subnet):
+    @patch('mmrazor.engine.runner.utils.check.export_fix_subnet')
+    @patch('mmrazor.models.task_modules.estimators.resource_estimator.'
+           'get_model_flops_params')
+    def test_run_loop(self, mock_flops, mock_export_fix_subnet):
         # test a new search: resume == None
         loop_cfg = copy.deepcopy(self.train_cfg)
         loop_cfg.runner = self.runner
@@ -169,16 +175,26 @@ def test_run(self, mock_export_fix_subnet):
         loop = LOOPS.build(loop_cfg)
         self.runner.rank = 0
         loop._epoch = 1
+
         fake_subnet = {'1': 'choice1', '2': 'choice2'}
         self.runner.work_dir = self.temp_dir
         loop.update_candidate_pool = MagicMock()
         loop.val_candidate_pool = MagicMock()
+
+        mutation_candidates = Candidates([fake_subnet] * loop.num_mutation)
+        for i in range(loop.num_mutation):
+            mutation_candidates.set_resources(i, 0.1 + 0.1 * i)
+            mutation_candidates.set_score(i, 99 + i)
+        crossover_candidates = Candidates([fake_subnet] * loop.num_crossover)
+        for i in range(loop.num_crossover):
+            crossover_candidates.set_resources(i, 0.1 + 0.1 * i)
+            crossover_candidates.set_score(i, 99 + i)
         loop.gen_mutation_candidates = \
-            MagicMock(return_value=[fake_subnet]*loop.num_mutation)
+            MagicMock(return_value=mutation_candidates)
         loop.gen_crossover_candidates = \
-            MagicMock(return_value=[fake_subnet]*loop.num_crossover)
-        loop.top_k_candidates = Candidates([(fake_subnet, 1.0),
-                                            (fake_subnet, 0.9)])
+            MagicMock(return_value=crossover_candidates)
+        loop.candidates = Candidates([fake_subnet] * 4)
+        mock_flops.return_value = (0.5, 101)
         mock_export_fix_subnet.return_value = fake_subnet
         loop.run()
         assert os.path.exists(
diff --git a/tests/test_runners/test_subnet_sampler_loop.py b/tests/test_runners/test_subnet_sampler_loop.py
index fca29b823..b2f6960e0 100644
--- a/tests/test_runners/test_subnet_sampler_loop.py
+++ b/tests/test_runners/test_subnet_sampler_loop.py
@@ -119,7 +119,7 @@ def setUp(self):
                 max_iters=12,
                 val_interval=2,
                 score_key='acc',
-                flops_range=None,
+                constraints_range=None,
                 num_candidates=4,
                 num_samples=2,
                 top_k=2,
@@ -190,7 +190,7 @@ def test_sample_subnet(self):
         loop._iter = loop.val_interval
         subnet = loop.sample_subnet()
         self.assertEqual(subnet, fake_subnet)
-        self.assertEqual(len(loop.top_k_candidates), loop.top_k - 1)
+        self.assertEqual(len(loop.top_k_candidates), loop.top_k)
 
     def test_run(self):
         # test run with _check_constraints
@@ -205,3 +205,8 @@ def test_run(self):
 
         self.assertEqual(runner.iter, runner.max_iters)
         assert os.path.exists(os.path.join(self.temp_dir, 'candidates.pkl'))
+
+
+if __name__ == '__main__':
+    import unittest
+    unittest.main()
diff --git a/tests/test_runners/test_utils/test_check.py b/tests/test_runners/test_utils/test_check.py
index b9bd57989..2f3a80eaa 100644
--- a/tests/test_runners/test_utils/test_check.py
+++ b/tests/test_runners/test_utils/test_check.py
@@ -1,7 +1,7 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 from unittest.mock import patch
 
-from mmrazor.engine.runner.utils import check_subnet_flops
+from mmrazor.engine.runner.utils import check_subnet_resources
 
 try:
     from mmdet.models.detectors import BaseDetector
@@ -12,29 +12,33 @@
 
 @patch('mmrazor.models.ResourceEstimator')
 @patch('mmrazor.models.SPOS')
-def test_check_subnet_flops(mock_model, mock_estimator):
-    # flops_range = None
-    flops_range = None
+def test_check_subnet_resources(mock_model, mock_estimator):
+    # constraints_range = dict()
+    constraints_range = dict()
     fake_subnet = {'1': 'choice1', '2': 'choice2'}
-    result = check_subnet_flops(mock_model, fake_subnet, mock_estimator,
-                                flops_range)
-    assert result is True
+    is_pass, _ = check_subnet_resources(mock_model, fake_subnet,
+                                        mock_estimator, constraints_range)
+    assert is_pass is True
 
-    # flops_range is not None
+    # constraints_range is not None
     # architecturte is BaseDetector
-    flops_range = (0., 100.)
+    constraints_range = dict(flops=(0, 330))
     mock_model.architecture = BaseDetector
     fake_results = {'flops': 50.}
     mock_estimator.estimate.return_value = fake_results
-    result = check_subnet_flops(mock_model, fake_subnet, mock_estimator,
-                                flops_range)
-    assert result is True
+    is_pass, _ = check_subnet_resources(
+        mock_model,
+        fake_subnet,
+        mock_estimator,
+        constraints_range,
+    )
+    assert is_pass is True
 
-    # flops_range is not None
+    # constraints_range is not None
     # architecturte is BaseDetector
-    flops_range = (0., 100.)
+    constraints_range = dict(flops=(0, 330))
     fake_results = {'flops': -50.}
     mock_estimator.estimate.return_value = fake_results
-    result = check_subnet_flops(mock_model, fake_subnet, mock_estimator,
-                                flops_range)
-    assert result is False
+    is_pass, _ = check_subnet_resources(mock_model, fake_subnet,
+                                        mock_estimator, constraints_range)
+    assert is_pass is False

From adccd52b9592a7d664e96f72943f9e5b8a40e351 Mon Sep 17 00:00:00 2001
From: aptsunny <aptsunny@tongji.edu.cn>
Date: Fri, 14 Oct 2022 15:26:52 +0800
Subject: [PATCH 03/18] clean code

---
 .gitignore | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index 9e3131fdb..e14966259 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,3 @@
-.base/*
-
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

From 3a3f91685d248cb92ece415f43a8ae36b08dbf3f Mon Sep 17 00:00:00 2001
From: aptsunny <aptsunny@tongji.edu.cn>
Date: Sat, 15 Oct 2022 23:52:54 +0800
Subject: [PATCH 04/18] update candidates

---
 .../engine/runner/evolution_search_loop.py    | 33 +++++++------------
 mmrazor/structures/subnet/candidate.py        | 19 ++++++++++-
 .../test_models/test_subnet/test_candidate.py | 21 ++++++++++++
 .../test_evolution_search_loop.py             |  4 +++
 4 files changed, 55 insertions(+), 22 deletions(-)

diff --git a/mmrazor/engine/runner/evolution_search_loop.py b/mmrazor/engine/runner/evolution_search_loop.py
index 692c0fc94..ed3f4dd2e 100644
--- a/mmrazor/engine/runner/evolution_search_loop.py
+++ b/mmrazor/engine/runner/evolution_search_loop.py
@@ -84,7 +84,6 @@ def __init__(self,
         self.num_candidates = num_candidates
         self.top_k = top_k
         self.constraints_range = constraints_range
-        self.key_indicator = list(self.constraints_range.keys())[0]
         self.estimator_cfg = resource_estimator_cfg
         self.score_key = score_key
         self.num_mutation = num_mutation
@@ -150,15 +149,14 @@ def run_epoch(self) -> None:
         self.runner.logger.info(f'top k scores after update: '
                                 f'{scores_after}')
 
-        self.candidates_mutator_crossover = Candidates()
         mutation_candidates = self.gen_mutation_candidates()
-        self.candidates_mutator_crossover.extend(mutation_candidates)
+        self.candidates_mutator_crossover = Candidates(mutation_candidates)
         crossover_candidates = self.gen_crossover_candidates()
         self.candidates_mutator_crossover.extend(crossover_candidates)
 
         assert len(self.candidates_mutator_crossover
                    ) <= self.num_candidates, 'Total of mutation and \
-            crossover should be no more than the number of candidates.'
+            crossover should be less than the number of candidates.'
 
         self.candidates = self.candidates_mutator_crossover
         self._epoch += 1
@@ -179,14 +177,13 @@ def sample_candidates(self) -> None:
             self.candidates = Candidates(self.candidates.data)
         else:
             self.candidates = Candidates([dict()] * self.num_candidates)
+        
+        if len(candidates_resources) > 0:
+            self.candidates.update_resources(candidates_resources)
         # broadcast candidates to val with multi-GPUs.
         broadcast_object_list(self.candidates.data)
         assert init_candidates + len(
             candidates_resources) == self.num_candidates
-        for i in range(len(candidates_resources)):
-            resources = candidates_resources[i][self.key_indicator] \
-                if len(candidates_resources[i]) != 0 else 0.
-            self.candidates.set_resources(i + init_candidates, resources)
 
     def update_candidates_scores(self) -> None:
         """Validate candicate one by one from the candicate pool, and update
@@ -200,9 +197,10 @@ def update_candidates_scores(self) -> None:
             self.runner.logger.info(
                 f'Epoch:[{self.runner.epoch}/{self.max_epochs}] '
                 f'Candidate:[{i + 1}/{self.num_candidates}] '
-                f'{self.key_indicator}: '
-                f'{self.candidates.resources(self.key_indicator)[i]} '
-                f'Score:{score}')
+                f'Flops: {self.candidates.resources("flops")[i]} '
+                f'Params: {self.candidates.resources("params")[i]} '
+                f'Latency: {self.candidates.resources("latency")[i]} '
+                f'Score:{self.candidates.scores}')
 
     def gen_mutation_candidates(self):
         """Generate specified number of mutation candicates."""
@@ -225,11 +223,7 @@ def gen_mutation_candidates(self):
                 mutation_resources.append(result)
 
         mutation_candidates = Candidates(mutation_candidates)
-
-        for i in range(self.num_mutation):
-            resources = mutation_resources[i][self.key_indicator] \
-                if len(mutation_resources[i]) != 0 else 0.
-            mutation_candidates.set_resources(i, resources)
+        mutation_candidates.update_resources(mutation_resources)
 
         return mutation_candidates
 
@@ -252,12 +246,9 @@ def gen_crossover_candidates(self):
             if is_pass:
                 crossover_candidates.append(crossover_candidate)
                 crossover_resources.append(result)
-        crossover_candidates = Candidates(crossover_candidates)
 
-        for i in range(self.num_crossover):
-            resources = crossover_resources[i][self.key_indicator] \
-                if len(crossover_resources[i]) != 0 else 0.
-            crossover_candidates.set_resources(i, resources)
+        crossover_candidates = Candidates(crossover_candidates)
+        crossover_candidates.update_resources(crossover_resources)
 
         return crossover_candidates
 
diff --git a/mmrazor/structures/subnet/candidate.py b/mmrazor/structures/subnet/candidate.py
index 44a1ad264..ca3030fe1 100644
--- a/mmrazor/structures/subnet/candidate.py
+++ b/mmrazor/structures/subnet/candidate.py
@@ -116,7 +116,11 @@ def _format_item(cond: Dict) -> Dict[str, Dict]:
     def append(self, item: _format_input) -> None:
         """Append operation."""
         item = self._format(item)
-        self.data.append(item)
+        if isinstance(item, list):
+            self.data = self.data + item 
+        else:
+            self.data.append(item)
+        # print(self.data)
 
     def insert(self, i: int, item: _format_input) -> None:
         """Insert operation."""
@@ -145,6 +149,19 @@ def set_resources(self,
         for _, value in self.data[i].items():
             value[key_indicator] = resources
 
+    def update_resources(self,
+                      resources: list, 
+                      mode: str= 'append') -> None:
+        """Update resources to the specified candidate."""
+        assert len(resources) <= len(self.data), 'Check the number of candidate resources.'
+        if mode == 'append':
+            start, end = len(self.data) - len(resources), len(self.data)
+        elif mode == 'insert':
+            start, end = 0, len(resources)
+        for i, item in enumerate(self.data[start:end]):
+            for _, value in item.items():
+                value.update(resources[i])
+
     def sort_by(self,
                 key_indicator: str = 'score',
                 reverse: bool = True) -> None:
diff --git a/tests/test_models/test_subnet/test_candidate.py b/tests/test_models/test_subnet/test_candidate.py
index fda3a3a02..7f30816ca 100644
--- a/tests/test_models/test_subnet/test_candidate.py
+++ b/tests/test_models/test_subnet/test_candidate.py
@@ -101,6 +101,23 @@ def test_set_resources(self):
         self.assertEqual(candidates.resources('flops'), [50., 49.9])
         self.assertEqual(candidates.resources('latency'), [0., 49.9])
 
+    def test_update_resources(self):
+        # test update_resources
+        candidates = Candidates([self.fake_subnet])
+        candidates.append([self.fake_subnet_with_score])
+        candidates_2 = Candidates(self.fake_subnet_with_resource)
+        candidates.append(candidates_2)
+        self.assertEqual(len(candidates), 3)
+        self.assertEqual(candidates.resources('flops'), [-1, 0., 50.])
+        self.assertEqual(candidates.resources('latency'), [-1, 0., 0.])
+        resources = [{'flops': -2}, {'latency': 4.}]
+        candidates.update_resources(resources)
+        self.assertEqual(candidates.resources('flops'), [-1, -2, 50.])
+        self.assertEqual(candidates.resources('latency'), [-1, 0., 4])
+        candidates.update_resources(resources, 'insert')
+        self.assertEqual(candidates.resources('flops'), [-2, -2, 50.])
+        self.assertEqual(candidates.resources('latency'), [-1, 4., 4.])
+
     def test_set_score(self):
         # test set_score
         candidates = Candidates([self.fake_subnet_with_score])
@@ -127,3 +144,7 @@ def test_sort(self):
         self.assertEqual(candidates.scores, [98., 99., 100.])
         candidates.sort_by(key_indicator='flops', reverse=False)
         self.assertEqual(candidates.scores, [100., 99., 98.])
+
+if __name__ == '__main__':
+    import unittest
+    unittest.main()
\ No newline at end of file
diff --git a/tests/test_runners/test_evolution_search_loop.py b/tests/test_runners/test_evolution_search_loop.py
index bb335d4f6..a3f03b970 100644
--- a/tests/test_runners/test_evolution_search_loop.py
+++ b/tests/test_runners/test_evolution_search_loop.py
@@ -210,3 +210,7 @@ def test_run_loop(self, mock_flops, mock_export_fix_subnet):
         self.runner.rank = 0
         loop.run()
         self.assertEqual(loop._max_epochs, 1)
+
+if __name__ == '__main__':
+    import unittest
+    unittest.main()
\ No newline at end of file

From 0bc1b1f81cc1c6f0e7db3d9b2751cd96d0eb384a Mon Sep 17 00:00:00 2001
From: aptsunny <aptsunny@tongji.edu.cn>
Date: Sun, 16 Oct 2022 00:08:50 +0800
Subject: [PATCH 05/18] clean

---
 mmrazor/engine/runner/evolution_search_loop.py   | 2 +-
 mmrazor/structures/subnet/candidate.py           | 3 +--
 tests/test_models/test_subnet/test_candidate.py  | 4 ----
 tests/test_runners/test_evolution_search_loop.py | 4 ----
 tests/test_runners/test_subnet_sampler_loop.py   | 5 -----
 5 files changed, 2 insertions(+), 16 deletions(-)

diff --git a/mmrazor/engine/runner/evolution_search_loop.py b/mmrazor/engine/runner/evolution_search_loop.py
index ed3f4dd2e..a228aad6e 100644
--- a/mmrazor/engine/runner/evolution_search_loop.py
+++ b/mmrazor/engine/runner/evolution_search_loop.py
@@ -177,7 +177,7 @@ def sample_candidates(self) -> None:
             self.candidates = Candidates(self.candidates.data)
         else:
             self.candidates = Candidates([dict()] * self.num_candidates)
-        
+
         if len(candidates_resources) > 0:
             self.candidates.update_resources(candidates_resources)
         # broadcast candidates to val with multi-GPUs.
diff --git a/mmrazor/structures/subnet/candidate.py b/mmrazor/structures/subnet/candidate.py
index ca3030fe1..0c2ea632f 100644
--- a/mmrazor/structures/subnet/candidate.py
+++ b/mmrazor/structures/subnet/candidate.py
@@ -120,7 +120,6 @@ def append(self, item: _format_input) -> None:
             self.data = self.data + item 
         else:
             self.data.append(item)
-        # print(self.data)
 
     def insert(self, i: int, item: _format_input) -> None:
         """Insert operation."""
@@ -150,7 +149,7 @@ def set_resources(self,
             value[key_indicator] = resources
 
     def update_resources(self,
-                      resources: list, 
+                      resources: list,
                       mode: str= 'append') -> None:
         """Update resources to the specified candidate."""
         assert len(resources) <= len(self.data), 'Check the number of candidate resources.'
diff --git a/tests/test_models/test_subnet/test_candidate.py b/tests/test_models/test_subnet/test_candidate.py
index 7f30816ca..9d23b6992 100644
--- a/tests/test_models/test_subnet/test_candidate.py
+++ b/tests/test_models/test_subnet/test_candidate.py
@@ -144,7 +144,3 @@ def test_sort(self):
         self.assertEqual(candidates.scores, [98., 99., 100.])
         candidates.sort_by(key_indicator='flops', reverse=False)
         self.assertEqual(candidates.scores, [100., 99., 98.])
-
-if __name__ == '__main__':
-    import unittest
-    unittest.main()
\ No newline at end of file
diff --git a/tests/test_runners/test_evolution_search_loop.py b/tests/test_runners/test_evolution_search_loop.py
index a3f03b970..bb335d4f6 100644
--- a/tests/test_runners/test_evolution_search_loop.py
+++ b/tests/test_runners/test_evolution_search_loop.py
@@ -210,7 +210,3 @@ def test_run_loop(self, mock_flops, mock_export_fix_subnet):
         self.runner.rank = 0
         loop.run()
         self.assertEqual(loop._max_epochs, 1)
-
-if __name__ == '__main__':
-    import unittest
-    unittest.main()
\ No newline at end of file
diff --git a/tests/test_runners/test_subnet_sampler_loop.py b/tests/test_runners/test_subnet_sampler_loop.py
index b2f6960e0..0ea55ebaf 100644
--- a/tests/test_runners/test_subnet_sampler_loop.py
+++ b/tests/test_runners/test_subnet_sampler_loop.py
@@ -205,8 +205,3 @@ def test_run(self):
 
         self.assertEqual(runner.iter, runner.max_iters)
         assert os.path.exists(os.path.join(self.temp_dir, 'candidates.pkl'))
-
-
-if __name__ == '__main__':
-    import unittest
-    unittest.main()

From fa140cfad37af454c1126d18a3a565e45ea85575 Mon Sep 17 00:00:00 2001
From: aptsunny <aptsunny@tongji.edu.cn>
Date: Sun, 16 Oct 2022 12:34:45 +0800
Subject: [PATCH 06/18] xx

---
 mmrazor/structures/subnet/candidate.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/mmrazor/structures/subnet/candidate.py b/mmrazor/structures/subnet/candidate.py
index 0c2ea632f..6cc598518 100644
--- a/mmrazor/structures/subnet/candidate.py
+++ b/mmrazor/structures/subnet/candidate.py
@@ -1,4 +1,6 @@
 # Copyright (c) OpenMMLab. All rights reserved.
+import warnings
+
 from collections import UserList
 from typing import Any, Dict, List, Optional, Union
 
@@ -157,6 +159,9 @@ def update_resources(self,
             start, end = len(self.data) - len(resources), len(self.data)
         elif mode == 'insert':
             start, end = 0, len(resources)
+        else:
+            start, end = 0, len(self.data)
+            warnings.warn('Please check whether the dimensions of A and B are matched')
         for i, item in enumerate(self.data[start:end]):
             for _, value in item.items():
                 value.update(resources[i])

From 952b4924b5f8ae3d39a4e342fa2bf61c62b895eb Mon Sep 17 00:00:00 2001
From: aptsunny <aptsunny@tongji.edu.cn>
Date: Mon, 17 Oct 2022 16:57:16 +0800
Subject: [PATCH 07/18] set_resource -> set_score

---
 .pre-commit-config.yaml                       | 18 +++----
 .../engine/runner/evolution_search_loop.py    | 15 ++++--
 mmrazor/engine/runner/subnet_sampler_loop.py  |  2 +-
 mmrazor/structures/subnet/candidate.py        | 26 +++-------
 .../test_models/test_subnet/test_candidate.py | 51 +++++++++++--------
 tests/test_runners/test_utils/test_check.py   |  4 ++
 6 files changed, 63 insertions(+), 53 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index fb1f746a4..48889ef92 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -30,15 +30,15 @@ repos:
     rev: v2.1.0
     hooks:
       - id: codespell
-  - repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.14
-    hooks:
-      - id: mdformat
-        args: ["--number"]
-        additional_dependencies:
-          - mdformat-gfm
-          - mdformat_frontmatter
-          - linkify-it-py
+  # - repo: https://github.com/executablebooks/mdformat
+  #   rev: 0.7.14
+  #   hooks:
+  #     - id: mdformat
+  #       args: ["--number"]
+  #       additional_dependencies:
+  #         - mdformat-gfm
+  #         - mdformat_frontmatter
+  #         - linkify-it-py
   - repo: https://github.com/myint/docformatter
     rev: v1.3.1
     hooks:
diff --git a/mmrazor/engine/runner/evolution_search_loop.py b/mmrazor/engine/runner/evolution_search_loop.py
index a228aad6e..b874547a0 100644
--- a/mmrazor/engine/runner/evolution_search_loop.py
+++ b/mmrazor/engine/runner/evolution_search_loop.py
@@ -41,6 +41,7 @@ class EvolutionSearchLoop(EpochBasedTrainLoop):
         num_crossover (int): The number of candidates got by crossover.
             Defaults to 25.
         mutate_prob (float): The probability of mutation. Defaults to 0.1.
+        crossover_prob (float): The probability of crossover. Defaults to 0.5.
         constraints_range (Dict[str, Any]): Constraints to be used for
             screening candidates. Defaults to dict(flops=(0, 330)).
         resource_estimator_cfg (Dict[str, Any]): Used for building a
@@ -64,6 +65,7 @@ def __init__(self,
                  num_mutation: int = 25,
                  num_crossover: int = 25,
                  mutate_prob: float = 0.1,
+                 crossover_prob: float = 0.5,
                  constraints_range: Dict[str, Any] = dict(flops=(0., 330.)),
                  resource_estimator_cfg: Dict[str, Any] = dict(),
                  score_key: str = 'accuracy/top1',
@@ -89,6 +91,7 @@ def __init__(self,
         self.num_mutation = num_mutation
         self.num_crossover = num_crossover
         self.mutate_prob = mutate_prob
+        self.crossover_prob = crossover_prob
         self.max_keep_ckpts = max_keep_ckpts
         self.resume_from = resume_from
 
@@ -179,7 +182,7 @@ def sample_candidates(self) -> None:
             self.candidates = Candidates([dict()] * self.num_candidates)
 
         if len(candidates_resources) > 0:
-            self.candidates.update_resources(candidates_resources)
+            self.candidates.update_resources(candidates_resources, start=len(self.candidates.data) - len(candidates_resources))
         # broadcast candidates to val with multi-GPUs.
         broadcast_object_list(self.candidates.data)
         assert init_candidates + len(
@@ -193,14 +196,14 @@ def update_candidates_scores(self) -> None:
             metrics = self._val_candidate()
             score = metrics[self.score_key] \
                 if len(metrics) != 0 else 0.
-            self.candidates.set_score(i, score)
+            self.candidates.set_resource(i, score, 'score')
             self.runner.logger.info(
                 f'Epoch:[{self.runner.epoch}/{self.max_epochs}] '
                 f'Candidate:[{i + 1}/{self.num_candidates}] '
                 f'Flops: {self.candidates.resources("flops")[i]} '
                 f'Params: {self.candidates.resources("params")[i]} '
                 f'Latency: {self.candidates.resources("latency")[i]} '
-                f'Score:{self.candidates.scores}')
+                f'Score: {self.candidates.scores} ')
 
     def gen_mutation_candidates(self):
         """Generate specified number of mutation candicates."""
@@ -263,7 +266,7 @@ def _crossover(self) -> SupportRandomSubnet:
         """Crossover."""
         candidate1 = random.choice(self.top_k_candidates.subnets)
         candidate2 = random.choice(self.top_k_candidates.subnets)
-        candidate = crossover(candidate1, candidate2)
+        candidate = crossover(candidate1, candidate2, prob=self.crossover_prob)
         return candidate
 
     def _resume(self):
@@ -347,3 +350,7 @@ def _check_constraints(self,
             return is_pass, results
         else:
             return is_pass
+
+if __name__ == '__main__':
+    import unittest
+    unittest.main()
\ No newline at end of file
diff --git a/mmrazor/engine/runner/subnet_sampler_loop.py b/mmrazor/engine/runner/subnet_sampler_loop.py
index cc89086da..508ad18a8 100644
--- a/mmrazor/engine/runner/subnet_sampler_loop.py
+++ b/mmrazor/engine/runner/subnet_sampler_loop.py
@@ -289,7 +289,7 @@ def update_candidates_scores(self) -> None:
             self.model.set_subnet(candidate)
             metrics = self._val_candidate()
             score = metrics[self.score_key] if len(metrics) != 0 else 0.
-            self.candidates.set_score(i, score)
+            self.candidates.set_resource(i, score, 'score')
 
     @torch.no_grad()
     def _val_candidate(self) -> Dict:
diff --git a/mmrazor/structures/subnet/candidate.py b/mmrazor/structures/subnet/candidate.py
index 6cc598518..3aef433f8 100644
--- a/mmrazor/structures/subnet/candidate.py
+++ b/mmrazor/structures/subnet/candidate.py
@@ -95,10 +95,9 @@ def _format(self, data: _format_input) -> _format_return:
                     that some indicators have been evaluated.
         """
 
-        def _format_item(cond: Dict) -> Dict[str, Dict]:
+        def _format_item(cond: Union[Dict[str, str], Dict[str, Dict]]) -> Dict[str, Dict]:
             """Transform Dict to Dict[str, Dict]."""
-            if len(cond.keys()) > 1 and isinstance(
-                    list(cond.values())[0], str):
+            if isinstance(list(cond.values())[0], str):
                 return {str(cond): {}.fromkeys(self._indicators, -1)}
             else:
                 for value in list(cond.values()):
@@ -136,32 +135,21 @@ def extend(self, other: Any) -> None:
         else:
             self.data.extend([other])
 
-    def set_score(self, i: int, score: float) -> None:
-        """Set score to the specified subnet by index."""
-        for _, value in self.data[i].items():
-            value['score'] = score
-
-    def set_resources(self,
+    def set_resource(self,
                       i: int,
                       resources: float,
                       key_indicator: str = 'flops') -> None:
         """Set resources to the specified subnet by index."""
-        assert key_indicator in ['flops', 'params', 'latency']
+        assert key_indicator in ['score', 'flops', 'params', 'latency']
         for _, value in self.data[i].items():
             value[key_indicator] = resources
 
     def update_resources(self,
                       resources: list,
-                      mode: str= 'append') -> None:
+                      start: int = 0) -> None:
         """Update resources to the specified candidate."""
-        assert len(resources) <= len(self.data), 'Check the number of candidate resources.'
-        if mode == 'append':
-            start, end = len(self.data) - len(resources), len(self.data)
-        elif mode == 'insert':
-            start, end = 0, len(resources)
-        else:
-            start, end = 0, len(self.data)
-            warnings.warn('Please check whether the dimensions of A and B are matched')
+        end = start + len(resources)
+        assert len(self.data) >= end, 'Check the number of candidate resources.'
         for i, item in enumerate(self.data[start:end]):
             for _, value in item.items():
                 value.update(resources[i])
diff --git a/tests/test_models/test_subnet/test_candidate.py b/tests/test_models/test_subnet/test_candidate.py
index 9d23b6992..2e1ed8097 100644
--- a/tests/test_models/test_subnet/test_candidate.py
+++ b/tests/test_models/test_subnet/test_candidate.py
@@ -26,6 +26,11 @@ def setUp(self) -> None:
                 'latency': 0.
             }
         }
+        self.has_flops_network = {
+            str(self.fake_subnet): {
+                'flops': 50.,
+            }
+        }
 
     def test_init(self):
         # initlist is None
@@ -38,6 +43,10 @@ def test_init(self):
         # initlist is UserList
         data = UserList([self.fake_subnet] * 2)
         self.assertEqual(len(candidates.data), 2)
+        self.assertEqual(candidates.resources('flops'), [-1, -1])
+        # initlist is list(Dict[str, Dict])
+        candidates = Candidates([self.has_flops_network] * 2)
+        self.assertEqual(candidates.resources('flops'), [50., 50.])
 
     def test_scores(self):
         # test property: scores
@@ -90,16 +99,26 @@ def test_extend(self):
         candidates.extend(candidates_2)
         self.assertEqual(len(candidates), 2)
 
-    def test_set_resources(self):
-        # test set_resources
+    def test_set_resource(self):
+        # test set_resource
         candidates = Candidates([self.fake_subnet])
         for kk in ['flops', 'params', 'latency']:
-            candidates.set_resources(0, 49.9, kk)
+            self.assertEqual(candidates.resources(kk)[0], -1)
+            candidates.set_resource(0, 49.9, kk)
             self.assertEqual(candidates.resources(kk)[0], 49.9)
         candidates.insert(0, self.fake_subnet_with_resource)
         self.assertEqual(len(candidates), 2)
         self.assertEqual(candidates.resources('flops'), [50., 49.9])
         self.assertEqual(candidates.resources('latency'), [0., 49.9])
+        # test set_score
+        candidates = Candidates([self.fake_subnet_with_score])
+        candidates.set_resource(0, 100.0, 'score')
+        self.assertEqual(candidates.scores[0], 100.)
+        candidates = Candidates([self.fake_subnet_with_score])
+        candidates.set_resource(0, 100.0, 'score')
+        candidates.extend(UserList([self.fake_subnet_with_resource]))
+        candidates.set_resource(1, 99.9, 'score')
+        self.assertEqual(candidates.scores, [100., 99.9])
 
     def test_update_resources(self):
         # test update_resources
@@ -111,32 +130,20 @@ def test_update_resources(self):
         self.assertEqual(candidates.resources('flops'), [-1, 0., 50.])
         self.assertEqual(candidates.resources('latency'), [-1, 0., 0.])
         resources = [{'flops': -2}, {'latency': 4.}]
-        candidates.update_resources(resources)
+        candidates.update_resources(resources, start=1)
         self.assertEqual(candidates.resources('flops'), [-1, -2, 50.])
         self.assertEqual(candidates.resources('latency'), [-1, 0., 4])
-        candidates.update_resources(resources, 'insert')
+        candidates.update_resources(resources, start=0)
         self.assertEqual(candidates.resources('flops'), [-2, -2, 50.])
         self.assertEqual(candidates.resources('latency'), [-1, 4., 4.])
 
-    def test_set_score(self):
-        # test set_score
-        candidates = Candidates([self.fake_subnet_with_score])
-        candidates.set_score(0, 100.0)
-        self.assertEqual(candidates.scores[0], 100.)
-
-        candidates = Candidates([self.fake_subnet_with_score])
-        candidates.set_score(0, 100.0)
-        candidates.extend(UserList([self.fake_subnet_with_resource]))
-        candidates.set_score(1, 99.9)
-        self.assertEqual(candidates.scores, [100., 99.9])
-
     def test_sort(self):
-        # test set_score
+        # test set_sort
         candidates = Candidates([self.fake_subnet_with_score])
         candidates.extend(UserList([self.fake_subnet_with_resource]))
         candidates.insert(0, self.fake_subnet)
-        candidates.set_score(0, 100.)
-        candidates.set_score(2, 98.)
+        candidates.set_resource(0, 100., 'score')
+        candidates.set_resource(2, 98., 'score')
         self.assertEqual(candidates.scores, [100., 99., 98.])
         candidates.sort_by(key_indicator='score', reverse=False)
         self.assertEqual(candidates.scores, [98., 99., 100.])
@@ -144,3 +151,7 @@ def test_sort(self):
         self.assertEqual(candidates.scores, [98., 99., 100.])
         candidates.sort_by(key_indicator='flops', reverse=False)
         self.assertEqual(candidates.scores, [100., 99., 98.])
+
+if __name__ == '__main__':
+    import unittest
+    unittest.main()
\ No newline at end of file
diff --git a/tests/test_runners/test_utils/test_check.py b/tests/test_runners/test_utils/test_check.py
index 2f3a80eaa..54b30c280 100644
--- a/tests/test_runners/test_utils/test_check.py
+++ b/tests/test_runners/test_utils/test_check.py
@@ -42,3 +42,7 @@ def test_check_subnet_resources(mock_model, mock_estimator):
     is_pass, _ = check_subnet_resources(mock_model, fake_subnet,
                                         mock_estimator, constraints_range)
     assert is_pass is False
+
+if __name__ == '__main__':
+    import unittest
+    unittest.main()
\ No newline at end of file

From 54b1b60ef018981364aadb39d594381c2627d0fe Mon Sep 17 00:00:00 2001
From: aptsunny <aptsunny@tongji.edu.cn>
Date: Mon, 17 Oct 2022 17:04:24 +0800
Subject: [PATCH 08/18] fix ci bug

---
 tests/test_models/test_subnet/test_candidate.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/tests/test_models/test_subnet/test_candidate.py b/tests/test_models/test_subnet/test_candidate.py
index 2e1ed8097..7f8bfe640 100644
--- a/tests/test_models/test_subnet/test_candidate.py
+++ b/tests/test_models/test_subnet/test_candidate.py
@@ -110,7 +110,6 @@ def test_set_resource(self):
         self.assertEqual(len(candidates), 2)
         self.assertEqual(candidates.resources('flops'), [50., 49.9])
         self.assertEqual(candidates.resources('latency'), [0., 49.9])
-        # test set_score
         candidates = Candidates([self.fake_subnet_with_score])
         candidates.set_resource(0, 100.0, 'score')
         self.assertEqual(candidates.scores[0], 100.)
@@ -151,7 +150,3 @@ def test_sort(self):
         self.assertEqual(candidates.scores, [98., 99., 100.])
         candidates.sort_by(key_indicator='flops', reverse=False)
         self.assertEqual(candidates.scores, [100., 99., 98.])
-
-if __name__ == '__main__':
-    import unittest
-    unittest.main()
\ No newline at end of file

From 00d1a8079c26fac7c0b772edeadfc9f26a04b5b4 Mon Sep 17 00:00:00 2001
From: aptsunny <aptsunny@tongji.edu.cn>
Date: Mon, 17 Oct 2022 17:05:53 +0800
Subject: [PATCH 09/18] py36 lint

---
 .pre-commit-config.yaml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 48889ef92..fb1f746a4 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -30,15 +30,15 @@ repos:
     rev: v2.1.0
     hooks:
       - id: codespell
-  # - repo: https://github.com/executablebooks/mdformat
-  #   rev: 0.7.14
-  #   hooks:
-  #     - id: mdformat
-  #       args: ["--number"]
-  #       additional_dependencies:
-  #         - mdformat-gfm
-  #         - mdformat_frontmatter
-  #         - linkify-it-py
+  - repo: https://github.com/executablebooks/mdformat
+    rev: 0.7.14
+    hooks:
+      - id: mdformat
+        args: ["--number"]
+        additional_dependencies:
+          - mdformat-gfm
+          - mdformat_frontmatter
+          - linkify-it-py
   - repo: https://github.com/myint/docformatter
     rev: v1.3.1
     hooks:

From cf1a47dde93e65e53f1e87d01f3510bb0224bed1 Mon Sep 17 00:00:00 2001
From: aptsunny <aptsunny@tongji.edu.cn>
Date: Mon, 17 Oct 2022 17:21:02 +0800
Subject: [PATCH 10/18] fix bug

---
 mmrazor/engine/runner/evolution_search_loop.py | 8 ++++----
 mmrazor/engine/runner/utils/check.py           | 4 ++--
 mmrazor/structures/subnet/candidate.py         | 4 +---
 3 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/mmrazor/engine/runner/evolution_search_loop.py b/mmrazor/engine/runner/evolution_search_loop.py
index b874547a0..97caed9ae 100644
--- a/mmrazor/engine/runner/evolution_search_loop.py
+++ b/mmrazor/engine/runner/evolution_search_loop.py
@@ -3,7 +3,7 @@
 import os.path as osp
 import random
 import warnings
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 import torch
 from mmengine import fileio
@@ -44,7 +44,7 @@ class EvolutionSearchLoop(EpochBasedTrainLoop):
         crossover_prob (float): The probability of crossover. Defaults to 0.5.
         constraints_range (Dict[str, Any]): Constraints to be used for
             screening candidates. Defaults to dict(flops=(0, 330)).
-        resource_estimator_cfg (Dict[str, Any]): Used for building a
+        resource_estimator_cfg (dict): Used for building a
             resource estimator. Defaults to dict().
         score_key (str): Specify one metric in evaluation results to score
             candidates. Defaults to 'accuracy_top-1'.
@@ -67,7 +67,7 @@ def __init__(self,
                  mutate_prob: float = 0.1,
                  crossover_prob: float = 0.5,
                  constraints_range: Dict[str, Any] = dict(flops=(0., 330.)),
-                 resource_estimator_cfg: Dict[str, Any] = dict(),
+                 resource_estimator_cfg: Dict = dict(),
                  score_key: str = 'accuracy/top1',
                  init_candidates: Optional[str] = None) -> None:
         super().__init__(runner, dataloader, max_epochs)
@@ -334,7 +334,7 @@ def _save_searcher_ckpt(self) -> None:
 
     def _check_constraints(self,
                            random_subnet: SupportRandomSubnet,
-                           need_feedback: bool = False):
+                           need_feedback: bool = False) -> Union[Tuple[bool, Dict], Dict]:
         """Check whether is beyond constraints.
 
         Returns:
diff --git a/mmrazor/engine/runner/utils/check.py b/mmrazor/engine/runner/utils/check.py
index d50ddc74b..0022655c2 100644
--- a/mmrazor/engine/runner/utils/check.py
+++ b/mmrazor/engine/runner/utils/check.py
@@ -1,6 +1,6 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import copy
-from typing import Any, Dict
+from typing import Any, Dict, Tuple, Union
 
 from mmrazor.models import ResourceEstimator
 from mmrazor.structures import export_fix_subnet, load_fix_subnet
@@ -18,7 +18,7 @@ def check_subnet_resources(model,
                            estimator: ResourceEstimator,
                            constraints_range: Dict[str,
                                                    Any] = dict(flops=(0,
-                                                                      330))):
+                                                                      330))) -> Union[Tuple[bool, Dict], Dict]:
     """Check whether is beyond resources constraints.
 
     Returns:
diff --git a/mmrazor/structures/subnet/candidate.py b/mmrazor/structures/subnet/candidate.py
index 3aef433f8..b0559463a 100644
--- a/mmrazor/structures/subnet/candidate.py
+++ b/mmrazor/structures/subnet/candidate.py
@@ -1,6 +1,4 @@
 # Copyright (c) OpenMMLab. All rights reserved.
-import warnings
-
 from collections import UserList
 from typing import Any, Dict, List, Optional, Union
 
@@ -95,7 +93,7 @@ def _format(self, data: _format_input) -> _format_return:
                     that some indicators have been evaluated.
         """
 
-        def _format_item(cond: Union[Dict[str, str], Dict[str, Dict]]) -> Dict[str, Dict]:
+        def _format_item(cond: Union[Dict, Dict[str, Dict]]) -> Dict[str, Dict]:
             """Transform Dict to Dict[str, Dict]."""
             if isinstance(list(cond.values())[0], str):
                 return {str(cond): {}.fromkeys(self._indicators, -1)}

From ad4f9dbbb28f9d12dc95022dde902a91217e4c71 Mon Sep 17 00:00:00 2001
From: aptsunny <aptsunny@tongji.edu.cn>
Date: Mon, 17 Oct 2022 18:22:30 +0800
Subject: [PATCH 11/18] fix check constrain

---
 .pre-commit-config.yaml                       |  9 --------
 .../engine/runner/evolution_search_loop.py    | 23 +++++++++----------
 mmrazor/engine/runner/subnet_sampler_loop.py  | 12 ++++------
 mmrazor/engine/runner/utils/check.py          | 16 ++++++-------
 mmrazor/structures/subnet/candidate.py        | 18 +++++++--------
 .../test_runners/test_subnet_sampler_loop.py  |  2 +-
 tests/test_runners/test_utils/test_check.py   |  3 ++-
 7 files changed, 35 insertions(+), 48 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index fb1f746a4..90529fe99 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -30,15 +30,6 @@ repos:
     rev: v2.1.0
     hooks:
       - id: codespell
-  - repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.14
-    hooks:
-      - id: mdformat
-        args: ["--number"]
-        additional_dependencies:
-          - mdformat-gfm
-          - mdformat_frontmatter
-          - linkify-it-py
   - repo: https://github.com/myint/docformatter
     rev: v1.3.1
     hooks:
diff --git a/mmrazor/engine/runner/evolution_search_loop.py b/mmrazor/engine/runner/evolution_search_loop.py
index 97caed9ae..6107a3fb6 100644
--- a/mmrazor/engine/runner/evolution_search_loop.py
+++ b/mmrazor/engine/runner/evolution_search_loop.py
@@ -173,7 +173,7 @@ def sample_candidates(self) -> None:
                 is_pass = False
                 candidate = self.model.sample_subnet()
                 is_pass, result = self._check_constraints(
-                    random_subnet=candidate, need_feedback=True)
+                    random_subnet=candidate)
                 if is_pass:
                     self.candidates.append(candidate)
                     candidates_resources.append(result)
@@ -182,7 +182,9 @@ def sample_candidates(self) -> None:
             self.candidates = Candidates([dict()] * self.num_candidates)
 
         if len(candidates_resources) > 0:
-            self.candidates.update_resources(candidates_resources, start=len(self.candidates.data) - len(candidates_resources))
+            self.candidates.update_resources(
+                candidates_resources,
+                start=len(self.candidates.data) - len(candidates_resources))
         # broadcast candidates to val with multi-GPUs.
         broadcast_object_list(self.candidates.data)
         assert init_candidates + len(
@@ -220,7 +222,7 @@ def gen_mutation_candidates(self):
 
             is_pass = False
             is_pass, result = self._check_constraints(
-                random_subnet=mutation_candidate, need_feedback=True)
+                random_subnet=mutation_candidate)
             if is_pass:
                 mutation_candidates.append(mutation_candidate)
                 mutation_resources.append(result)
@@ -245,7 +247,7 @@ def gen_crossover_candidates(self):
 
             is_pass = False
             is_pass, result = self._check_constraints(
-                random_subnet=crossover_candidate, need_feedback=True)
+                random_subnet=crossover_candidate)
             if is_pass:
                 crossover_candidates.append(crossover_candidate)
                 crossover_resources.append(result)
@@ -332,9 +334,8 @@ def _save_searcher_ckpt(self) -> None:
                     if osp.isfile(ckpt_path):
                         os.remove(ckpt_path)
 
-    def _check_constraints(self,
-                           random_subnet: SupportRandomSubnet,
-                           need_feedback: bool = False) -> Union[Tuple[bool, Dict], Dict]:
+    def _check_constraints(
+            self, random_subnet: SupportRandomSubnet) -> Tuple[bool, Dict]:
         """Check whether is beyond constraints.
 
         Returns:
@@ -346,11 +347,9 @@ def _check_constraints(self,
             estimator=self.estimator,
             constraints_range=self.constraints_range)
 
-        if need_feedback:
-            return is_pass, results
-        else:
-            return is_pass
+        return is_pass, results
+
 
 if __name__ == '__main__':
     import unittest
-    unittest.main()
\ No newline at end of file
+    unittest.main()
diff --git a/mmrazor/engine/runner/subnet_sampler_loop.py b/mmrazor/engine/runner/subnet_sampler_loop.py
index 508ad18a8..d51d0de76 100644
--- a/mmrazor/engine/runner/subnet_sampler_loop.py
+++ b/mmrazor/engine/runner/subnet_sampler_loop.py
@@ -275,7 +275,8 @@ def get_candidates_with_sample(self,
         for _ in range(num_samples):
             if random.random() >= self.cur_prob or len(self.candidates) == 0:
                 subnet = self._sample_from_supernet()
-                if self._check_constraints(subnet, need_feedback=False):
+                is_pass, _ = self._check_constraints(subnet)
+                if is_pass:
                     sampled_candidates.append(subnet)
                 num_sample_from_supernet += 1
             else:
@@ -312,9 +313,7 @@ def _sample_from_candidates(self) -> SupportRandomSubnet:
         subnet = random.choice(self.candidates.data)
         return subnet
 
-    def _check_constraints(self,
-                           random_subnet: SupportRandomSubnet,
-                           need_feedback: bool = False):
+    def _check_constraints(self, random_subnet: SupportRandomSubnet):
         """Check whether is beyond constraints.
 
         Returns:
@@ -326,10 +325,7 @@ def _check_constraints(self,
             estimator=self.estimator,
             constraints_range=self.constraints_range)
 
-        if need_feedback:
-            return is_pass, results
-        else:
-            return is_pass
+        return is_pass, results
 
     def _save_candidates(self) -> None:
         """Save the candidates to init the next searching."""
diff --git a/mmrazor/engine/runner/utils/check.py b/mmrazor/engine/runner/utils/check.py
index 0022655c2..d094b0a47 100644
--- a/mmrazor/engine/runner/utils/check.py
+++ b/mmrazor/engine/runner/utils/check.py
@@ -1,6 +1,6 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import copy
-from typing import Any, Dict, Tuple, Union
+from typing import Any, Dict, Tuple
 
 from mmrazor.models import ResourceEstimator
 from mmrazor.structures import export_fix_subnet, load_fix_subnet
@@ -13,19 +13,19 @@
     BaseDetector = get_placeholder('mmdet')
 
 
-def check_subnet_resources(model,
-                           subnet: SupportRandomSubnet,
-                           estimator: ResourceEstimator,
-                           constraints_range: Dict[str,
-                                                   Any] = dict(flops=(0,
-                                                                      330))) -> Union[Tuple[bool, Dict], Dict]:
+def check_subnet_resources(
+    model,
+    subnet: SupportRandomSubnet,
+    estimator: ResourceEstimator,
+    constraints_range: Dict[str, Any] = dict(flops=(0, 330))
+) -> Tuple[bool, Dict]:
     """Check whether is beyond resources constraints.
 
     Returns:
         bool: The result of checking.
     """
     if constraints_range is None:
-        return True, None
+        return True, dict()
 
     assert hasattr(model, 'set_subnet') and hasattr(model, 'architecture')
     model.set_subnet(subnet)
diff --git a/mmrazor/structures/subnet/candidate.py b/mmrazor/structures/subnet/candidate.py
index b0559463a..bba75c242 100644
--- a/mmrazor/structures/subnet/candidate.py
+++ b/mmrazor/structures/subnet/candidate.py
@@ -93,7 +93,8 @@ def _format(self, data: _format_input) -> _format_return:
                     that some indicators have been evaluated.
         """
 
-        def _format_item(cond: Union[Dict, Dict[str, Dict]]) -> Dict[str, Dict]:
+        def _format_item(
+                cond: Union[Dict, Dict[str, Dict]]) -> Dict[str, Dict]:
             """Transform Dict to Dict[str, Dict]."""
             if isinstance(list(cond.values())[0], str):
                 return {str(cond): {}.fromkeys(self._indicators, -1)}
@@ -116,7 +117,7 @@ def append(self, item: _format_input) -> None:
         """Append operation."""
         item = self._format(item)
         if isinstance(item, list):
-            self.data = self.data + item 
+            self.data = self.data + item
         else:
             self.data.append(item)
 
@@ -134,20 +135,19 @@ def extend(self, other: Any) -> None:
             self.data.extend([other])
 
     def set_resource(self,
-                      i: int,
-                      resources: float,
-                      key_indicator: str = 'flops') -> None:
+                     i: int,
+                     resources: float,
+                     key_indicator: str = 'flops') -> None:
         """Set resources to the specified subnet by index."""
         assert key_indicator in ['score', 'flops', 'params', 'latency']
         for _, value in self.data[i].items():
             value[key_indicator] = resources
 
-    def update_resources(self,
-                      resources: list,
-                      start: int = 0) -> None:
+    def update_resources(self, resources: list, start: int = 0) -> None:
         """Update resources to the specified candidate."""
         end = start + len(resources)
-        assert len(self.data) >= end, 'Check the number of candidate resources.'
+        assert len(
+            self.data) >= end, 'Check the number of candidate resources.'
         for i, item in enumerate(self.data[start:end]):
             for _, value in item.items():
                 value.update(resources[i])
diff --git a/tests/test_runners/test_subnet_sampler_loop.py b/tests/test_runners/test_subnet_sampler_loop.py
index 0ea55ebaf..1c9422fc1 100644
--- a/tests/test_runners/test_subnet_sampler_loop.py
+++ b/tests/test_runners/test_subnet_sampler_loop.py
@@ -200,7 +200,7 @@ def test_run(self):
         fake_subnet = {'1': 'choice1', '2': 'choice2'}
         runner.model.sample_subnet = MagicMock(return_value=fake_subnet)
         loop = runner.build_train_loop(cfg.train_cfg)
-        loop._check_constraints = MagicMock(return_value=True)
+        loop._check_constraints = MagicMock(return_value=(True, dict()))
         runner.train()
 
         self.assertEqual(runner.iter, runner.max_iters)
diff --git a/tests/test_runners/test_utils/test_check.py b/tests/test_runners/test_utils/test_check.py
index 54b30c280..12d97c053 100644
--- a/tests/test_runners/test_utils/test_check.py
+++ b/tests/test_runners/test_utils/test_check.py
@@ -43,6 +43,7 @@ def test_check_subnet_resources(mock_model, mock_estimator):
                                         mock_estimator, constraints_range)
     assert is_pass is False
 
+
 if __name__ == '__main__':
     import unittest
-    unittest.main()
\ No newline at end of file
+    unittest.main()

From c314033e9286e54b0c72c28c4bab806e7d031965 Mon Sep 17 00:00:00 2001
From: aptsunny <aptsunny@tongji.edu.cn>
Date: Mon, 17 Oct 2022 19:30:13 +0800
Subject: [PATCH 12/18] py36 ci

---
 .pre-commit-config.yaml                          | 9 +++++++++
 tests/test_runners/test_evolution_search_loop.py | 8 ++++----
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 90529fe99..fb1f746a4 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -30,6 +30,15 @@ repos:
     rev: v2.1.0
     hooks:
       - id: codespell
+  - repo: https://github.com/executablebooks/mdformat
+    rev: 0.7.14
+    hooks:
+      - id: mdformat
+        args: ["--number"]
+        additional_dependencies:
+          - mdformat-gfm
+          - mdformat_frontmatter
+          - linkify-it-py
   - repo: https://github.com/myint/docformatter
     rev: v1.3.1
     hooks:
diff --git a/tests/test_runners/test_evolution_search_loop.py b/tests/test_runners/test_evolution_search_loop.py
index bb335d4f6..4c8f46f02 100644
--- a/tests/test_runners/test_evolution_search_loop.py
+++ b/tests/test_runners/test_evolution_search_loop.py
@@ -183,12 +183,12 @@ def test_run_loop(self, mock_flops, mock_export_fix_subnet):
 
         mutation_candidates = Candidates([fake_subnet] * loop.num_mutation)
         for i in range(loop.num_mutation):
-            mutation_candidates.set_resources(i, 0.1 + 0.1 * i)
-            mutation_candidates.set_score(i, 99 + i)
+            mutation_candidates.set_resource(i, 0.1 + 0.1 * i, 'flops')
+            mutation_candidates.set_resource(i, 99 + i, 'score')
         crossover_candidates = Candidates([fake_subnet] * loop.num_crossover)
         for i in range(loop.num_crossover):
-            crossover_candidates.set_resources(i, 0.1 + 0.1 * i)
-            crossover_candidates.set_score(i, 99 + i)
+            crossover_candidates.set_resource(i, 0.1 + 0.1 * i, 'flops')
+            crossover_candidates.set_resource(i, 99 + i, 'score')
         loop.gen_mutation_candidates = \
             MagicMock(return_value=mutation_candidates)
         loop.gen_crossover_candidates = \

From 6afafeafa284e55cd154aac5761245803c7cd163 Mon Sep 17 00:00:00 2001
From: aptsunny <aptsunny@tongji.edu.cn>
Date: Tue, 18 Oct 2022 16:56:30 +0800
Subject: [PATCH 13/18] redesign candidate

---
 mmrazor/engine/runner/evolution_search_loop.py |  5 -----
 mmrazor/engine/runner/utils/check.py           |  2 +-
 mmrazor/structures/subnet/candidate.py         | 10 +++++++---
 tests/test_runners/test_utils/test_check.py    |  5 -----
 4 files changed, 8 insertions(+), 14 deletions(-)

diff --git a/mmrazor/engine/runner/evolution_search_loop.py b/mmrazor/engine/runner/evolution_search_loop.py
index 6107a3fb6..4aea0f94d 100644
--- a/mmrazor/engine/runner/evolution_search_loop.py
+++ b/mmrazor/engine/runner/evolution_search_loop.py
@@ -348,8 +348,3 @@ def _check_constraints(
             constraints_range=self.constraints_range)
 
         return is_pass, results
-
-
-if __name__ == '__main__':
-    import unittest
-    unittest.main()
diff --git a/mmrazor/engine/runner/utils/check.py b/mmrazor/engine/runner/utils/check.py
index d094b0a47..203abfd88 100644
--- a/mmrazor/engine/runner/utils/check.py
+++ b/mmrazor/engine/runner/utils/check.py
@@ -22,7 +22,7 @@ def check_subnet_resources(
     """Check whether is beyond resources constraints.
 
     Returns:
-        bool: The result of checking.
+        bool, result: The result of checking.      
     """
     if constraints_range is None:
         return True, dict()
diff --git a/mmrazor/structures/subnet/candidate.py b/mmrazor/structures/subnet/candidate.py
index bba75c242..ca0c8236e 100644
--- a/mmrazor/structures/subnet/candidate.py
+++ b/mmrazor/structures/subnet/candidate.py
@@ -96,13 +96,13 @@ def _format(self, data: _format_input) -> _format_return:
         def _format_item(
                 cond: Union[Dict, Dict[str, Dict]]) -> Dict[str, Dict]:
             """Transform Dict to Dict[str, Dict]."""
-            if isinstance(list(cond.values())[0], str):
-                return {str(cond): {}.fromkeys(self._indicators, -1)}
-            else:
+            if isinstance(list(cond.values())[0], dict):
                 for value in list(cond.values()):
                     for key in list(self._indicators):
                         value.setdefault(key, 0.)
                 return cond
+            else:
+                return {str(cond): {}.fromkeys(self._indicators, -1)}
 
         if isinstance(data, UserList):
             return [_format_item(i) for i in data.data]
@@ -134,6 +134,10 @@ def extend(self, other: Any) -> None:
         else:
             self.data.extend([other])
 
+    def set_score(self, i: int, score: float) -> None:
+        """Set score to the specified subnet by index."""
+        self.set_resource(i, score, 'score')
+
     def set_resource(self,
                      i: int,
                      resources: float,
diff --git a/tests/test_runners/test_utils/test_check.py b/tests/test_runners/test_utils/test_check.py
index 12d97c053..2f3a80eaa 100644
--- a/tests/test_runners/test_utils/test_check.py
+++ b/tests/test_runners/test_utils/test_check.py
@@ -42,8 +42,3 @@ def test_check_subnet_resources(mock_model, mock_estimator):
     is_pass, _ = check_subnet_resources(mock_model, fake_subnet,
                                         mock_estimator, constraints_range)
     assert is_pass is False
-
-
-if __name__ == '__main__':
-    import unittest
-    unittest.main()

From 20633ddde4a2f83b4d4f364740fa0e45b36c6cf3 Mon Sep 17 00:00:00 2001
From: aptsunny <aptsunny@tongji.edu.cn>
Date: Tue, 18 Oct 2022 17:01:58 +0800
Subject: [PATCH 14/18] fix pre-commit

---
 mmrazor/engine/runner/utils/check.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mmrazor/engine/runner/utils/check.py b/mmrazor/engine/runner/utils/check.py
index 203abfd88..b1c581708 100644
--- a/mmrazor/engine/runner/utils/check.py
+++ b/mmrazor/engine/runner/utils/check.py
@@ -22,7 +22,7 @@ def check_subnet_resources(
     """Check whether is beyond resources constraints.
 
     Returns:
-        bool, result: The result of checking.      
+        bool, result: The result of checking.
     """
     if constraints_range is None:
         return True, dict()

From 1bc80b88d034bb349a337c2e24b7097c5eea94e0 Mon Sep 17 00:00:00 2001
From: aptsunny <aptsunny@tongji.edu.cn>
Date: Tue, 18 Oct 2022 20:56:57 +0800
Subject: [PATCH 15/18] update cfg

---
 .../spos/spos_mobilenet_search_8xb128_in1k.py |  2 +-
 .../engine/runner/evolution_search_loop.py    | 25 +++++++++----------
 mmrazor/engine/runner/subnet_sampler_loop.py  | 17 +++++++------
 3 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/configs/nas/mmcls/spos/spos_mobilenet_search_8xb128_in1k.py b/configs/nas/mmcls/spos/spos_mobilenet_search_8xb128_in1k.py
index 4f5edb316..87553ec39 100644
--- a/configs/nas/mmcls/spos/spos_mobilenet_search_8xb128_in1k.py
+++ b/configs/nas/mmcls/spos/spos_mobilenet_search_8xb128_in1k.py
@@ -13,5 +13,5 @@
     num_mutation=25,
     num_crossover=25,
     mutate_prob=0.1,
-    flops_range=(0., 465.),
+    constraints_range=dict(flops=(0., 465.)),
     score_key='accuracy/top1')
diff --git a/mmrazor/engine/runner/evolution_search_loop.py b/mmrazor/engine/runner/evolution_search_loop.py
index 4aea0f94d..158cee710 100644
--- a/mmrazor/engine/runner/evolution_search_loop.py
+++ b/mmrazor/engine/runner/evolution_search_loop.py
@@ -14,7 +14,7 @@
 from torch.utils.data import DataLoader
 
 from mmrazor.models.task_modules import ResourceEstimator
-from mmrazor.registry import LOOPS
+from mmrazor.registry import LOOPS, TASK_UTILS
 from mmrazor.structures import Candidates, export_fix_subnet
 from mmrazor.utils import SupportRandomSubnet
 from .utils import check_subnet_resources, crossover
@@ -44,8 +44,8 @@ class EvolutionSearchLoop(EpochBasedTrainLoop):
         crossover_prob (float): The probability of crossover. Defaults to 0.5.
         constraints_range (Dict[str, Any]): Constraints to be used for
             screening candidates. Defaults to dict(flops=(0, 330)).
-        resource_estimator_cfg (dict): Used for building a
-            resource estimator. Defaults to dict().
+        resource_estimator_cfg (dict, Optional): Used for building a
+            resource estimator. Defaults to None.
         score_key (str): Specify one metric in evaluation results to score
             candidates. Defaults to 'accuracy_top-1'.
         init_candidates (str, optional): The candidates file path, which is
@@ -67,7 +67,7 @@ def __init__(self,
                  mutate_prob: float = 0.1,
                  crossover_prob: float = 0.5,
                  constraints_range: Dict[str, Any] = dict(flops=(0., 330.)),
-                 resource_estimator_cfg: Dict = dict(),
+                 resource_estimator_cfg: Optional[Dict] = None,
                  score_key: str = 'accuracy/top1',
                  init_candidates: Optional[str] = None) -> None:
         super().__init__(runner, dataloader, max_epochs)
@@ -86,7 +86,6 @@ def __init__(self,
         self.num_candidates = num_candidates
         self.top_k = top_k
         self.constraints_range = constraints_range
-        self.estimator_cfg = resource_estimator_cfg
         self.score_key = score_key
         self.num_mutation = num_mutation
         self.num_crossover = num_crossover
@@ -103,7 +102,10 @@ def __init__(self,
                 correct init candidates file'
 
         self.top_k_candidates = Candidates()
-        self.estimator = ResourceEstimator(**resource_estimator_cfg)
+        if 'type' in resource_estimator_cfg:
+            self.estimator = TASK_UTILS.build(resource_estimator_cfg)
+        else:
+            self.estimator = ResourceEstimator(**resource_estimator_cfg)
 
         if self.runner.distributed:
             self.model = runner.model.module
@@ -170,7 +172,6 @@ def sample_candidates(self) -> None:
         init_candidates = len(self.candidates)
         if self.runner.rank == 0:
             while len(self.candidates) < self.num_candidates:
-                is_pass = False
                 candidate = self.model.sample_subnet()
                 is_pass, result = self._check_constraints(
                     random_subnet=candidate)
@@ -200,7 +201,7 @@ def update_candidates_scores(self) -> None:
                 if len(metrics) != 0 else 0.
             self.candidates.set_resource(i, score, 'score')
             self.runner.logger.info(
-                f'Epoch:[{self.runner.epoch}/{self.max_epochs}] '
+                f'Epoch:[{self._epoch}/{self._max_epochs}] '
                 f'Candidate:[{i + 1}/{self.num_candidates}] '
                 f'Flops: {self.candidates.resources("flops")[i]} '
                 f'Params: {self.candidates.resources("params")[i]} '
@@ -220,7 +221,6 @@ def gen_mutation_candidates(self):
 
             mutation_candidate = self._mutation()
 
-            is_pass = False
             is_pass, result = self._check_constraints(
                 random_subnet=mutation_candidate)
             if is_pass:
@@ -245,7 +245,6 @@ def gen_crossover_candidates(self):
 
             crossover_candidate = self._crossover()
 
-            is_pass = False
             is_pass, result = self._check_constraints(
                 random_subnet=crossover_candidate)
             if is_pass:
@@ -278,7 +277,7 @@ def _resume(self):
             for k in searcher_resume.keys():
                 setattr(self, k, searcher_resume[k])
             epoch_start = int(searcher_resume['_epoch'])
-            self._max_epochs = self.max_epochs - epoch_start
+            self._max_epochs = self._max_epochs - epoch_start
             self.runner.logger.info('#' * 100)
             self.runner.logger.info(f'Resume from epoch: {epoch_start}')
             self.runner.logger.info('#' * 100)
@@ -322,11 +321,11 @@ def _save_searcher_ckpt(self) -> None:
                 osp.join(self.runner.work_dir,
                          f'search_epoch_{self._epoch}.pkl'))
             self.runner.logger.info(
-                f'Epoch:[{self._epoch}/{self.max_epochs}], top1_score: '
+                f'Epoch:[{self._epoch}/{self._max_epochs}], top1_score: '
                 f'{self.top_k_candidates.scores[0]}')
 
             if self.max_keep_ckpts > 0:
-                cur_ckpt = self.runner.epoch + 1
+                cur_ckpt = self._epoch + 1
                 redundant_ckpts = range(1, cur_ckpt - self.max_keep_ckpts)
                 for _step in redundant_ckpts:
                     ckpt_path = osp.join(self.runner.work_dir,
diff --git a/mmrazor/engine/runner/subnet_sampler_loop.py b/mmrazor/engine/runner/subnet_sampler_loop.py
index d51d0de76..d853c4723 100644
--- a/mmrazor/engine/runner/subnet_sampler_loop.py
+++ b/mmrazor/engine/runner/subnet_sampler_loop.py
@@ -3,7 +3,7 @@
 import os
 import random
 from abc import abstractmethod
-from typing import Any, Dict, List, Sequence, Tuple, Union
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
 
 import torch
 from mmengine import fileio
@@ -13,7 +13,7 @@
 from torch.utils.data import DataLoader
 
 from mmrazor.models.task_modules import ResourceEstimator
-from mmrazor.registry import LOOPS
+from mmrazor.registry import LOOPS, TASK_UTILS
 from mmrazor.structures import Candidates
 from mmrazor.utils import SupportRandomSubnet
 from .utils import check_subnet_resources
@@ -101,9 +101,8 @@ class GreedySamplerTrainLoop(BaseSamplerTrainLoop):
             candidates. Defaults to 'accuracy_top-1'.
         constraints_range (Dict[str, Any]): Constraints to be used for
             screening candidates. Defaults to dict(flops=(0, 330)).
-        resource_estimator_cfg (dict): The config for building estimator, which
-            is be used to estimate the flops of sampled subnet. Defaults to
-            None, which means default config is used. Defaults to dict().
+        resource_estimator_cfg (dict, Optional): Used for building a
+            resource estimator. Defaults to None.
         num_candidates (int): The number of the candidates consist of samples
             from supernet and itself. Defaults to 1000.
         num_samples (int): The number of sample in each sampling subnet.
@@ -138,7 +137,7 @@ def __init__(self,
                  val_interval: int = 1000,
                  score_key: str = 'accuracy/top1',
                  constraints_range: Dict[str, Any] = dict(flops=(0, 330)),
-                 resource_estimator_cfg: Dict[str, Any] = dict(),
+                 resource_estimator_cfg: Optional[Dict] = None,
                  num_candidates: int = 1000,
                  num_samples: int = 10,
                  top_k: int = 5,
@@ -175,7 +174,11 @@ def __init__(self,
 
         self.candidates = Candidates()
         self.top_k_candidates = Candidates()
-        self.estimator = ResourceEstimator(**resource_estimator_cfg)
+        if 'type' in resource_estimator_cfg:
+            self.estimator = TASK_UTILS.build(resource_estimator_cfg)
+        else:
+            self.estimator = ResourceEstimator(**resource_estimator_cfg)
+
 
     def run(self) -> None:
         """Launch training."""

From 9a37974aa1a1784fffc9473522caa8e98ce4dc74 Mon Sep 17 00:00:00 2001
From: aptsunny <aptsunny@tongji.edu.cn>
Date: Tue, 18 Oct 2022 21:29:49 +0800
Subject: [PATCH 16/18] add build_resource_estimator

---
 .../spos_shufflenet_search_8xb128_in1k.py     |  2 +-
 .../detnas_frcnn_shufflenet_search_coco_1x.py |  2 +-
 .../engine/runner/evolution_search_loop.py    | 46 +++++++++++++++++--
 mmrazor/engine/runner/subnet_sampler_loop.py  | 41 ++++++++++++++++-
 4 files changed, 83 insertions(+), 8 deletions(-)

diff --git a/configs/nas/mmcls/spos/spos_shufflenet_search_8xb128_in1k.py b/configs/nas/mmcls/spos/spos_shufflenet_search_8xb128_in1k.py
index f3f963e40..f5a5e88f4 100644
--- a/configs/nas/mmcls/spos/spos_shufflenet_search_8xb128_in1k.py
+++ b/configs/nas/mmcls/spos/spos_shufflenet_search_8xb128_in1k.py
@@ -13,5 +13,5 @@
     num_mutation=25,
     num_crossover=25,
     mutate_prob=0.1,
-    flops_range=(0., 330.),
+    constraints_range=dict(flops=(0, 330)),
     score_key='accuracy/top1')
diff --git a/configs/nas/mmdet/detnas/detnas_frcnn_shufflenet_search_coco_1x.py b/configs/nas/mmdet/detnas/detnas_frcnn_shufflenet_search_coco_1x.py
index d1dd1637a..689618362 100644
--- a/configs/nas/mmdet/detnas/detnas_frcnn_shufflenet_search_coco_1x.py
+++ b/configs/nas/mmdet/detnas/detnas_frcnn_shufflenet_search_coco_1x.py
@@ -13,5 +13,5 @@
     num_mutation=20,
     num_crossover=20,
     mutate_prob=0.1,
-    flops_range=(0., 300.),
+    constraints_range=dict(flops=(0, 330)),
     score_key='coco/bbox_mAP')
diff --git a/mmrazor/engine/runner/evolution_search_loop.py b/mmrazor/engine/runner/evolution_search_loop.py
index 158cee710..551240fc7 100644
--- a/mmrazor/engine/runner/evolution_search_loop.py
+++ b/mmrazor/engine/runner/evolution_search_loop.py
@@ -1,4 +1,5 @@
 # Copyright (c) OpenMMLab. All rights reserved.
+import copy
 import os
 import os.path as osp
 import random
@@ -102,16 +103,53 @@ def __init__(self,
                 correct init candidates file'
 
         self.top_k_candidates = Candidates()
-        if 'type' in resource_estimator_cfg:
-            self.estimator = TASK_UTILS.build(resource_estimator_cfg)
-        else:
-            self.estimator = ResourceEstimator(**resource_estimator_cfg)
 
         if self.runner.distributed:
             self.model = runner.model.module
         else:
             self.model = runner.model
 
+        # Build resource estimator.
+        resource_estimator_cfg = dict() if resource_estimator_cfg is None else resource_estimator_cfg
+        self.estimator = self.build_resource_estimator(resource_estimator_cfg)
+
+    def build_resource_estimator(
+            self, resource_estimator: Union[ResourceEstimator, Dict]) -> ResourceEstimator:
+        """Build resource estimator for search loop.
+
+        Examples of ``resource_estimator``:
+
+            # `ResourceEstimator` will be used
+            resource_estimator = dict()
+            
+            # custom resource_estimator
+            resource_estimator = dict(type='mmrazor.ResourceEstimator')
+        
+        Args:
+            resource_estimator (ResourceEstimator or dict): A resource_estimator or a dict
+            to build resource estimator. If ``resource_estimator`` is a resource estimator
+            object, just returns itself.
+        
+        Returns:
+            :obj:`ResourceEstimator`: Resource estimator object build from
+            ``resource_estimator``.
+        """
+        if isinstance(resource_estimator, ResourceEstimator):
+            return resource_estimator
+        elif not isinstance(resource_estimator, dict):
+            raise TypeError(
+                'resource estimator should be a ResourceEstimator object or dict, but'
+                f'got {resource_estimator}')
+
+        resource_estimator_cfg = copy.deepcopy(resource_estimator)  # type: ignore
+
+        if 'type' in resource_estimator_cfg:
+            estimator = TASK_UTILS.build(resource_estimator_cfg)
+        else:
+            estimator = ResourceEstimator(**resource_estimator_cfg)  # type: ignore
+
+        return estimator  # type: ignore
+
     def run(self) -> None:
         """Launch searching."""
         self.runner.call_hook('before_train')
diff --git a/mmrazor/engine/runner/subnet_sampler_loop.py b/mmrazor/engine/runner/subnet_sampler_loop.py
index d853c4723..5bb5bf0cb 100644
--- a/mmrazor/engine/runner/subnet_sampler_loop.py
+++ b/mmrazor/engine/runner/subnet_sampler_loop.py
@@ -1,4 +1,5 @@
 # Copyright (c) OpenMMLab. All rights reserved.
+import copy
 import math
 import os
 import random
@@ -174,11 +175,47 @@ def __init__(self,
 
         self.candidates = Candidates()
         self.top_k_candidates = Candidates()
+
+        # Build resource estimator.
+        resource_estimator_cfg = dict() if resource_estimator_cfg is None else resource_estimator_cfg
+        self.estimator = self.build_resource_estimator(resource_estimator_cfg)
+
+    def build_resource_estimator(
+            self, resource_estimator: Union[ResourceEstimator, Dict]) -> ResourceEstimator:
+        """Build resource estimator for search loop.
+
+        Examples of ``resource_estimator``:
+
+            # `ResourceEstimator` will be used
+            resource_estimator = dict()
+            
+            # custom resource_estimator
+            resource_estimator = dict(type='mmrazor.ResourceEstimator')
+        
+        Args:
+            resource_estimator (ResourceEstimator or dict): A resource_estimator or a dict
+            to build resource estimator. If ``resource_estimator`` is a resource estimator
+            object, just returns itself.
+        
+        Returns:
+            :obj:`ResourceEstimator`: Resource estimator object build from
+            ``resource_estimator``.
+        """
+        if isinstance(resource_estimator, ResourceEstimator):
+            return resource_estimator
+        elif not isinstance(resource_estimator, dict):
+            raise TypeError(
+                'resource estimator should be a ResourceEstimator object or dict, but'
+                f'got {resource_estimator}')
+
+        resource_estimator_cfg = copy.deepcopy(resource_estimator)  # type: ignore
+
         if 'type' in resource_estimator_cfg:
-            self.estimator = TASK_UTILS.build(resource_estimator_cfg)
+            estimator = TASK_UTILS.build(resource_estimator_cfg)
         else:
-            self.estimator = ResourceEstimator(**resource_estimator_cfg)
+            estimator = ResourceEstimator(**resource_estimator_cfg)  # type: ignore
 
+        return estimator  # type: ignore
 
     def run(self) -> None:
         """Launch training."""

From 24a9ebe711e34f6dc4570087a63c1fa16c4625fb Mon Sep 17 00:00:00 2001
From: aptsunny <aptsunny@tongji.edu.cn>
Date: Tue, 18 Oct 2022 21:40:40 +0800
Subject: [PATCH 17/18] fix ci bug

---
 .../engine/runner/evolution_search_loop.py    | 29 +++++++++++--------
 mmrazor/engine/runner/subnet_sampler_loop.py  | 29 +++++++++++--------
 2 files changed, 34 insertions(+), 24 deletions(-)

diff --git a/mmrazor/engine/runner/evolution_search_loop.py b/mmrazor/engine/runner/evolution_search_loop.py
index 551240fc7..009280778 100644
--- a/mmrazor/engine/runner/evolution_search_loop.py
+++ b/mmrazor/engine/runner/evolution_search_loop.py
@@ -110,26 +110,29 @@ def __init__(self,
             self.model = runner.model
 
         # Build resource estimator.
-        resource_estimator_cfg = dict() if resource_estimator_cfg is None else resource_estimator_cfg
+        resource_estimator_cfg = dict(
+        ) if resource_estimator_cfg is None else resource_estimator_cfg
         self.estimator = self.build_resource_estimator(resource_estimator_cfg)
 
     def build_resource_estimator(
-            self, resource_estimator: Union[ResourceEstimator, Dict]) -> ResourceEstimator:
+        self, resource_estimator: Union[ResourceEstimator,
+                                        Dict]) -> ResourceEstimator:
         """Build resource estimator for search loop.
 
         Examples of ``resource_estimator``:
 
             # `ResourceEstimator` will be used
             resource_estimator = dict()
-            
+
             # custom resource_estimator
             resource_estimator = dict(type='mmrazor.ResourceEstimator')
-        
+
         Args:
-            resource_estimator (ResourceEstimator or dict): A resource_estimator or a dict
-            to build resource estimator. If ``resource_estimator`` is a resource estimator
-            object, just returns itself.
-        
+            resource_estimator (ResourceEstimator or dict): A
+            resource_estimator or a dict to build resource estimator.
+            If ``resource_estimator`` is a resource estimator object,
+            just returns itself.
+
         Returns:
             :obj:`ResourceEstimator`: Resource estimator object build from
             ``resource_estimator``.
@@ -138,15 +141,17 @@ def build_resource_estimator(
             return resource_estimator
         elif not isinstance(resource_estimator, dict):
             raise TypeError(
-                'resource estimator should be a ResourceEstimator object or dict, but'
-                f'got {resource_estimator}')
+                'resource estimator should be a ResourceEstimator object or'
+                f'dict, but got {resource_estimator}')
 
-        resource_estimator_cfg = copy.deepcopy(resource_estimator)  # type: ignore
+        resource_estimator_cfg = copy.deepcopy(
+            resource_estimator)  # type: ignore
 
         if 'type' in resource_estimator_cfg:
             estimator = TASK_UTILS.build(resource_estimator_cfg)
         else:
-            estimator = ResourceEstimator(**resource_estimator_cfg)  # type: ignore
+            estimator = ResourceEstimator(
+                **resource_estimator_cfg)  # type: ignore
 
         return estimator  # type: ignore
 
diff --git a/mmrazor/engine/runner/subnet_sampler_loop.py b/mmrazor/engine/runner/subnet_sampler_loop.py
index 5bb5bf0cb..273561568 100644
--- a/mmrazor/engine/runner/subnet_sampler_loop.py
+++ b/mmrazor/engine/runner/subnet_sampler_loop.py
@@ -177,26 +177,29 @@ def __init__(self,
         self.top_k_candidates = Candidates()
 
         # Build resource estimator.
-        resource_estimator_cfg = dict() if resource_estimator_cfg is None else resource_estimator_cfg
+        resource_estimator_cfg = dict(
+        ) if resource_estimator_cfg is None else resource_estimator_cfg
         self.estimator = self.build_resource_estimator(resource_estimator_cfg)
 
     def build_resource_estimator(
-            self, resource_estimator: Union[ResourceEstimator, Dict]) -> ResourceEstimator:
+        self, resource_estimator: Union[ResourceEstimator,
+                                        Dict]) -> ResourceEstimator:
         """Build resource estimator for search loop.
 
         Examples of ``resource_estimator``:
 
             # `ResourceEstimator` will be used
             resource_estimator = dict()
-            
+
             # custom resource_estimator
             resource_estimator = dict(type='mmrazor.ResourceEstimator')
-        
+
         Args:
-            resource_estimator (ResourceEstimator or dict): A resource_estimator or a dict
-            to build resource estimator. If ``resource_estimator`` is a resource estimator
-            object, just returns itself.
-        
+            resource_estimator (ResourceEstimator or dict):
+            A resource_estimator or a dict to build resource estimator.
+            If ``resource_estimator`` is a resource estimator object,
+            just returns itself.
+
         Returns:
             :obj:`ResourceEstimator`: Resource estimator object build from
             ``resource_estimator``.
@@ -205,15 +208,17 @@ def build_resource_estimator(
             return resource_estimator
         elif not isinstance(resource_estimator, dict):
             raise TypeError(
-                'resource estimator should be a ResourceEstimator object or dict, but'
-                f'got {resource_estimator}')
+                'resource estimator should be a ResourceEstimator object or'
+                f'dict, but got {resource_estimator}')
 
-        resource_estimator_cfg = copy.deepcopy(resource_estimator)  # type: ignore
+        resource_estimator_cfg = copy.deepcopy(
+            resource_estimator)  # type: ignore
 
         if 'type' in resource_estimator_cfg:
             estimator = TASK_UTILS.build(resource_estimator_cfg)
         else:
-            estimator = ResourceEstimator(**resource_estimator_cfg)  # type: ignore
+            estimator = ResourceEstimator(
+                **resource_estimator_cfg)  # type: ignore
 
         return estimator  # type: ignore
 

From 534ed4338727e9d22909384fbfae3429e7786492 Mon Sep 17 00:00:00 2001
From: aptsunny <aptsunny@tongji.edu.cn>
Date: Tue, 18 Oct 2022 22:04:57 +0800
Subject: [PATCH 18/18] remove runner.epoch in testcase

---
 tests/test_runners/test_evolution_search_loop.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_runners/test_evolution_search_loop.py b/tests/test_runners/test_evolution_search_loop.py
index 4c8f46f02..6d8814a7b 100644
--- a/tests/test_runners/test_evolution_search_loop.py
+++ b/tests/test_runners/test_evolution_search_loop.py
@@ -122,7 +122,6 @@ def test_run_epoch(self, flops_params, mock_export_fix_subnet,
         loop_cfg.runner = self.runner
         loop_cfg.dataloader = self.dataloader
         loop_cfg.evaluator = self.evaluator
-        loop_cfg.runner.epoch = 1
         loop = LOOPS.build(loop_cfg)
         self.runner.rank = 0
         self.runner.distributed = False