[Enhancement] Add benchmark test script (open-mmlab#262)

* update benchmark test

* fix circle ci gpu config

* fix lints

Co-authored-by: gaojianfei <gaojianfei@sensetime.com>

.circleci/test.yml

@@ -124,8 +124,7 @@ jobs:
             docker exec mmrazor pip install -e /mmdetection
             docker exec mmrazor pip install -e /mmclassification
             docker exec mmrazor pip install -e /mmsegmentation
-            pip install -r requirements.txt
-            python -c 'import mmcv; print(mmcv.__version__)'
+            docker exec mmrazor pip install -r requirements.txt
       - run:
           name: Build and install
           command: |

.dev_scripts/benchmark_test.py

@@ -40,6 +40,8 @@ def parse_args():
         '--work-dir',
         default='work_dirs/benchmark_test',
         help='the dir to save metric')
+    parser.add_argument(
+        '--replace-ceph', action='store_true', help='load data from ceph')
     parser.add_argument(
         '--run', action='store_true', help='run script directly')
     parser.add_argument(
@@ -66,12 +68,70 @@ def parse_args():
     return args
 
 
+def replace_to_ceph(cfg):
+
+    file_client_args = dict(
+        backend='petrel',
+        path_mapping=dict({
+            './data/coco':
+            's3://openmmlab/datasets/detection/coco',
+            'data/coco':
+            's3://openmmlab/datasets/detection/coco',
+            './data/cityscapes':
+            's3://openmmlab/datasets/segmentation/cityscapes',
+            'data/cityscapes':
+            's3://openmmlab/datasets/segmentation/cityscapes',
+            './data/imagenet':
+            's3://openmmlab/datasets/classification/imagenet',
+            'data/imagenet':
+            's3://openmmlab/datasets/classification/imagenet',
+        }))
+
+    def _process_pipeline(dataset, name):
+
+        def replace_img(pipeline):
+            if pipeline['type'] == 'LoadImageFromFile':
+                pipeline['file_client_args'] = file_client_args
+
+        def replace_ann(pipeline):
+            if pipeline['type'] == 'LoadAnnotations' or pipeline[
+                    'type'] == 'LoadPanopticAnnotations':
+                pipeline['file_client_args'] = file_client_args
+
+        if 'pipeline' in dataset:
+            replace_img(dataset.pipeline[0])
+            replace_ann(dataset.pipeline[1])
+            if 'dataset' in dataset:
+                # dataset wrapper
+                replace_img(dataset.dataset.pipeline[0])
+                replace_ann(dataset.dataset.pipeline[1])
+        else:
+            # dataset wrapper
+            replace_img(dataset.dataset.pipeline[0])
+            replace_ann(dataset.dataset.pipeline[1])
+
+    def _process_evaluator(evaluator, name):
+        if evaluator['type'] == 'CocoPanopticMetric':
+            evaluator['file_client_args'] = file_client_args
+
+    # half ceph
+    _process_pipeline(cfg.train_dataloader.dataset, cfg.filename)
+    _process_pipeline(cfg.val_dataloader.dataset, cfg.filename)
+    _process_pipeline(cfg.test_dataloader.dataset, cfg.filename)
+    _process_evaluator(cfg.val_evaluator, cfg.filename)
+    _process_evaluator(cfg.test_evaluator, cfg.filename)
+
+
 def create_test_job_batch(commands, model_info, args, port):
 
     fname = model_info.name
 
-    config = Path(model_info.config)
-    # assert config.exists(), f'{fname}: {config} not found.'
+    cfg_path = Path(model_info.config)
+
+    cfg = mmengine.Config.fromfile(cfg_path)
+
+    if args.replace_ceph:
+        replace_to_ceph(cfg)
 
     http_prefix = 'https://download.openmmlab.com/mmrazor/'
     if 's3://' in args.checkpoint_root:
@@ -99,6 +159,8 @@ def create_test_job_batch(commands, model_info, args, port):
     job_name = f'{args.job_name}_{fname}'
     work_dir = Path(args.work_dir) / fname
     work_dir.mkdir(parents=True, exist_ok=True)
+    test_cfg_path = work_dir / 'config.py'
+    cfg.dump(test_cfg_path)
 
     if args.quotatype is not None:
         quota_cfg = f'#SBATCH --quotatype {args.quotatype}\n'
@@ -110,24 +172,24 @@ def create_test_job_batch(commands, model_info, args, port):
     master_port = f'NASTER_PORT={port}'
 
     script_name = osp.join('tools', 'test.py')
-    job_script = (
-        f'#!/bin/bash\n'
-        f'#SBATCH --output {work_dir}/job.%j.out\n'
-        f'#SBATCH --partition={args.partition}\n'
-        f'#SBATCH --job-name {job_name}\n'
-        f'#SBATCH --gres=gpu:{args.gpus}\n'
-        f'{quota_cfg}'
-        f'#SBATCH --ntasks-per-node={args.gpus}\n'
-        f'#SBATCH --ntasks={args.gpus}\n'
-        f'#SBATCH --cpus-per-task=5\n\n'
-        f'{master_port} {runner} -u {script_name} {config} {checkpoint} '
-        f'--work-dir {work_dir} '
-        f'--launcher={launcher}\n')
+    job_script = (f'#!/bin/bash\n'
+                  f'#SBATCH --output {work_dir}/job.%j.out\n'
+                  f'#SBATCH --partition={args.partition}\n'
+                  f'#SBATCH --job-name {job_name}\n'
+                  f'#SBATCH --gres=gpu:{args.gpus}\n'
+                  f'{quota_cfg}'
+                  f'#SBATCH --ntasks-per-node={args.gpus}\n'
+                  f'#SBATCH --ntasks={args.gpus}\n'
+                  f'#SBATCH --cpus-per-task=5\n\n'
+                  f'{master_port} {runner} -u {script_name} '
+                  f'{test_cfg_path} {checkpoint} '
+                  f'--work-dir {work_dir} '
+                  f'--launcher={launcher}\n')
 
     with open(work_dir / 'job.sh', 'w') as f:
         f.write(job_script)
 
-    commands.append(f'echo "{config}"')
+    commands.append(f'echo "{test_cfg_path}"')
     if args.local:
         commands.append(f'bash {work_dir}/job.sh')
     else:
@@ -176,9 +238,8 @@ def summary(args):
         expect_result = model_info.results[0].metrics
         summary_result = {
             'expect': expect_result,
-            'actual':
-            {METRIC_MAPPINGS[k]: v
-             for k, v in latest_result.items()}
+            'actual': {k: v
+                       for k, v in latest_result.items()}
         }
         model_results[model_name] = summary_result
 

configs/distill/mmcls/abloss/README.md

@@ -10,7 +10,6 @@ An activation boundary for a neuron refers to a separating hyperplane that deter
 
 <img width="1184" alt="pipeline" src="https://user-images.githubusercontent.com/88702197/187422794-d681ed58-293a-4d9e-9e5b-9937289136a7.png">
 
-
 ## Results and models
 
 ### Classification

configs/distill/mmcls/byot/README.md

@@ -8,7 +8,6 @@ Convolutional neural networks have been widely deployed in various application s
 
 ![byot](https://user-images.githubusercontent.com/88702197/187422992-e7bd692d-b6d4-44d8-8b36-741e0cf1c4f6.png)
 
-
 ## Results and models
 
 #### Classification

configs/distill/mmcls/dafl/README.md

@@ -10,7 +10,6 @@ Learning portable neural networks is very essential for computer vision for the
 
 <img width="910" alt="pipeline" src="https://user-images.githubusercontent.com/88702197/187423163-b34896fc-8516-403b-acd7-4c0b8e43af5b.png">
 
-
 ## Results and models
 
 ### Classification

configs/distill/mmcls/fitnet/README.md

@@ -22,7 +22,6 @@ almost 10.4 times less parameters outperforms a larger, state-of-the-art teacher
 
 <img width="743" alt="pipeline" src="https://user-images.githubusercontent.com/88702197/187423686-68719140-a978-4a19-a684-42b1d793d1fb.png">
 
-
 ## Results and models
 
 ### Classification

configs/distill/mmdet/cwd/README.md

@@ -10,7 +10,6 @@ Knowledge distillation (KD) has been proven to be a simple and effective tool fo
 
 ![pipeline](https://user-images.githubusercontent.com/88702197/187424502-d8efb7a3-c40c-4e53-a36c-bd947de464a4.png)
 
-
 ## Results and models
 
 ### Segmentation

configs/distill/mmdet/fbkd/README.md

@@ -10,7 +10,6 @@ Knowledge distillation, in which a student model is trained to mimic a teacher m
 
 <img width="836" alt="pipeline" src="https://user-images.githubusercontent.com/88702197/187424617-6259a7fc-b610-40ae-92eb-f21450dcbaa1.png">
 
-
 ## Results and models
 
 ### Detection

configs/nas/mmcls/spos/README.md

@@ -11,7 +11,6 @@ Comprehensive experiments verify that our approach is flexible and effective. It
 
 ![pipeline](https://user-images.githubusercontent.com/88702197/187424862-c2f3fde1-4a48-4eda-9ff7-c65971b683ba.jpg)
 
-
 ## Introduction
 
 ### Supernet pre-training on ImageNet

configs/pruning/mmcls/autoslim/README.md

@@ -11,8 +11,6 @@ Notably, by setting optimized channel numbers, our AutoSlim-MobileNet-v2 at 305M
 
 ![pipeline](https://user-images.githubusercontent.com/88702197/187425354-d90e4b36-e033-4dc0-b951-64a536e61b71.png)
 
-
-
 ## Introduction
 
 ### Supernet pre-training on ImageNet

docs/en/advanced_guides/algorithm.md

@@ -1,4 +1,5 @@
 # Algorithm
+
 ## Introduction
 
 ### What is algorithm in MMRazor
@@ -10,15 +11,15 @@ MMRazor is a model compression toolkit, which includes 4 mianstream technologies
 - Knowledge Distillation (KD)
 - Quantization (come soon)
 
-And in MMRazor, `algorithm` is a general item for these technologies. For example, in NAS,  
+And in MMRazor, `algorithm` is a general item for these technologies. For example, in NAS,
 
-[SPOS](https://github.com/open-mmlab/mmrazor/blob/master/configs/nas/spos)[ ](https://arxiv.org/abs/1904.00420)is an `algorithm`,  [CWD](https://github.com/open-mmlab/mmrazor/blob/master/configs/distill/cwd) is also an `algorithm` of knowledge distillation. 
+[SPOS](https://github.com/open-mmlab/mmrazor/blob/master/configs/nas/spos)[ ](https://arxiv.org/abs/1904.00420)is an `algorithm`,  [CWD](https://github.com/open-mmlab/mmrazor/blob/master/configs/distill/cwd) is also an `algorithm` of knowledge distillation.
 
-`algorithm`  is the entrance of `mmrazor/models` . Its role in MMRazor is the same as both `classifier` in [MMClassification](https://github.com/open-mmlab/mmclassification) and `detector` in [MMDetection](https://github.com/open-mmlab/mmdetection). 
+`algorithm`  is the entrance of `mmrazor/models` . Its role in MMRazor is the same as both `classifier` in [MMClassification](https://github.com/open-mmlab/mmclassification) and `detector` in [MMDetection](https://github.com/open-mmlab/mmdetection).
 
 ### About base algorithm
 
-In the directory of `models/algorith``ms`, all model compression algorithms are divided into 4 subdirectories: nas / pruning / distill / quantization. These algorithms must inherit from `BaseAlgorithm`, whose definition is as below.
+In the directory of ``` models/algorith``ms ```, all model compression algorithms are divided into 4 subdirectories: nas / pruning / distill / quantization. These algorithms must inherit from `BaseAlgorithm`, whose definition is as below.
 
 ```Python
 from typing import Dict, List, Optional, Tuple, Union
@@ -37,17 +38,17 @@ class BaseAlgorithm(BaseModel):
                  architecture: Union[BaseModel, Dict],
                  data_preprocessor: Optional[Union[Dict, nn.Module]] = None,
                  init_cfg: Optional[Dict] = None):
-        
+
         ......
-        
+
         super().__init__(data_preprocessor, init_cfg)
         self.architecture = architecture
 
     def forward(self,
                 batch_inputs: torch.Tensor,
                 data_samples: Optional[List[BaseDataElement]] = None,
                 mode: str = 'tensor') -> ForwardResults:
-        
+
         if mode == 'loss':
             return self.loss(batch_inputs, data_samples)
         elif mode == 'tensor':
@@ -86,12 +87,10 @@ class BaseAlgorithm(BaseModel):
 
 As you can see from above, `BaseAlgorithm` is inherited from `BaseModel` of MMEngine. `BaseModel` implements the basic functions of the algorithmic model, such as weights initialize,
 
-batch inputs preprocess (see more information in `BaseDataPreprocessor` class of MMEngine), parse losses, and update model parameters. For more details of `BaseModel` , you can see docs for `BaseModel`. 
+batch inputs preprocess (see more information in `BaseDataPreprocessor` class of MMEngine), parse losses, and update model parameters. For more details of `BaseModel` , you can see docs for `BaseModel`.
 
 `BaseAlgorithm`'s forward is just a wrapper of `BaseModel`'s forward. Sub-classes inherited from BaseAlgorithm only need to override the `loss` method, which implements the logic to calculate loss, thus various algorithms can be trained in the runner.
 
-
-
 ## How to use existing algorithms in MMRazor
 
 1. Configure your architecture that will be slimmed
@@ -108,7 +107,7 @@ architecture = _base_.model
 
 - Use your customized model as below, which is an example of defining a VGG model as our architecture.
 
-> How to customize architectures can refer to our tutorial: [Customize Architectures](https://mmrazor.readthedocs.io/en/latest/tutorials/Tutorial_3_customize_architectures.html#).
+> How to customize architectures can refer to our tutorial: [Customize Architectures](https://mmrazor.readthedocs.io/en/latest/tutorials/Tutorial_3_customize_architectures.html).
 
 ```Python
 default_scope='mmcls'
@@ -123,7 +122,7 @@ architecture = dict(
     ))
 ```
 
-2. Apply the registered algorithm to your architecture. 
+2. Apply the registered algorithm to your architecture.
 
 > The arg name of `algorithm` in config is **model** rather than **algorithm** in order to get better supports of MMCV and MMEngine.
 
@@ -196,8 +195,8 @@ class XXX(BaseAlgorithm):
     def __init__(self, architecture):
         super().__init__(architecture)
         ......
-        
-    def loss(self, batch_inputs): 
+
+    def loss(self, batch_inputs):
         ......
         return LossResults
 ```
@@ -219,17 +218,17 @@ class XXX(BaseAlgorithm):
     def loss(self, batch_inputs):
         ......
         return LossResults
-    
+
     def aaa(self):
         ......
-        
+
     def bbb(self):
         ......
 ```
 
 4. Import the class
 
-You can add the following line to `mmrazor/models/algorithms/``{subdirectory}/``__init__.py`
+You can add the following line to ``` mmrazor/models/algorithms/``{subdirectory}/``__init__.py ```
 
 ```CoffeeScript
 from .xxx import XXX
@@ -255,12 +254,12 @@ Please refer to our tutorials about how to customize different algorithms for mo
 
 1. NAS
 
-[Customize NAS algorithms](https://mmrazor.readthedocs.io/en/latest/tutorials/Tutorial_4_customize_nas_algorithms.html#)
+[Customize NAS algorithms](https://mmrazor.readthedocs.io/en/latest/tutorials/Tutorial_4_customize_nas_algorithms.html)
 
 2. Pruning
 
 [Customize Pruning algorithms](https://mmrazor.readthedocs.io/en/latest/tutorials/Tutorial_5_customize_pruning_algorithms.html)
 
-3. Distill 
+3. Distill
 
-[Customize KD algorithms](https://mmrazor.readthedocs.io/en/latest/tutorials/Tutorial_6_customize_kd_algorithms.html)
+[Customize KD algorithms](https://mmrazor.readthedocs.io/en/latest/tutorials/Tutorial_6_customize_kd_algorithms.html)

docs/en/advanced_guides/apply_existing_algorithms_to_new_tasks.md

@@ -1,11 +1,12 @@
 # Apply existing algorithms to new tasks
+
 Here we show how to apply existing algorithms to other tasks with an example of [SPOS ](https://github.com/open-mmlab/mmrazor/tree/dev-1.x/configs/nas/mmcls/spos)& [DetNAS](https://github.com/open-mmlab/mmrazor/tree/dev-1.x/configs/nas/mmdet/detnas).
 
 > SPOS: Single Path One-Shot NAS for classification
 >
 > DetNAS: Single Path One-Shot NAS for detection
 
-**You just need to configure the existing algorithms in your config only by replacing** **the architecture of** **mmcls** **with** **mmdet****'s**
+**You just need to configure the existing algorithms in your config only by replacing** **the architecture of** **mmcls** **with** **mmdet**\*\*'s\*\*
 
 You can implement a new algorithm by inheriting from the existing algorithm quickly if the new task's specificity leads to the failure of applying directly.
 
@@ -79,4 +80,4 @@ model = dict(
     mutator=dict(type='mmrazor.OneShotModuleMutator'))
 
 find_unused_parameters = True
-```
+```