[Improvement] Adapt OFA series with SearchableMobileNetV3 (#385)

* fix mutable bug in AttentiveMobileNetV3

* remove unness code

* update ATTENTIVE_SUBNET_A0-A6.yaml with optimized names

* unify the sampling usage in sandwich_rule-based NAS

* use alias to export subnet

* update OFA configs

* fix attr bug

* fix comments

* update convert_supernet2subnet.py

* correct the way to dump DerivedMutable

* fix convert index bug

* update OFA configs & models

* fix dynamic2static

* generalize convert_ofa_ckpt.py

* update input_resizer

* update README.md

* fix ut

* update export_fix_subnet

* update _dynamic_to_static

* update fix_subnet UT & minor fix bugs

* fix ut

* add new autoaug compared to attentivenas

* clean

* fix act

* fix act_cfg

* update fix_subnet

* fix lint

* add docstring

Co-authored-by: gaoyang07 <1546308416@qq.com>
Co-authored-by: aptsunny <aptsunny@tongji.edu.cn>

configs/_base_/nas_backbones/attentive_mobilenetv3_supernet.py

@@ -40,12 +40,8 @@
         [1792, 1984, 1984 - 1792],  # last layer
     ])
 
-_INPUT_MUTABLE = dict(
-    input_resizer=dict(type='DynamicInputResizer'),
-    mutable_shape=dict(
-        type='OneShotMutableValue',
-        value_list=[[192, 192], [224, 224], [256, 256], [288, 288]],
-        default_value=[224, 224]))
+input_resizer_cfg = dict(
+    input_sizes=[[192, 192], [224, 224], [256, 256], [288, 288]])
 
 nas_backbone = dict(
     type='AttentiveMobileNetV3',

configs/_base_/nas_backbones/ofa_mobilenetv3_supernet.py

@@ -37,12 +37,21 @@
         [1024, 1280, 1280 - 1024],  # last layer
     ])
 
+input_resizer_cfg = dict(
+    input_sizes=[[128, 128], [140, 140], [144, 144], [152, 152], [192, 192],
+                 [204, 204], [224, 224], [256, 256]])
+
 nas_backbone = dict(
     type='mmrazor.AttentiveMobileNetV3',
     arch_setting=arch_setting,
     out_indices=(6, ),
     stride_list=[1, 2, 2, 2, 1, 2],
     with_se_list=[False, False, True, False, True, True],
+    act_cfg_list=[
+        'HSwish', 'ReLU', 'ReLU', 'ReLU', 'HSwish', 'HSwish', 'HSwish',
+        'HSwish', 'HSwish'
+    ],
     conv_cfg=dict(type='OFAConv2d'),
-    norm_cfg=dict(type='mmrazor.DynamicBatchNorm2d', momentum=0.0),
+    norm_cfg=dict(type='mmrazor.DynamicBatchNorm2d', momentum=0.1),
+    fine_grained_mode=True,
     with_attentive_shortcut=False)

configs/_base_/settings/imagenet_bs2048_bignas.py

@@ -145,7 +145,11 @@
             prob=1.0,
             magnitude=9,
             extra_params=extra_params),
-        dict(type='ShearY', prob=0.6, magnitude=3, extra_params=extra_params),
+        dict(
+            type='mmrazor.ShearY',
+            prob=0.6,
+            magnitude=3,
+            extra_params=extra_params),
     ],
     [
         dict(

configs/_base_/settings/imagenet_bs2048_ofa.py

@@ -0,0 +1,98 @@
+# dataset settings
+dataset_type = 'mmcls.ImageNet'
+
+# data preprocessor
+data_preprocessor = dict(
+    type='mmcls.ClsDataPreprocessor',
+    # RGB format normalization parameters
+    mean=[123.675, 116.28, 103.53],
+    std=[58.395, 57.12, 57.375],
+    # convert image from BGR to RGB
+    to_rgb=True,
+)
+
+bgr_mean = data_preprocessor['mean'][::-1]
+bgr_std = data_preprocessor['std'][::-1]
+
+extra_params = dict(
+    translate_const=int(224 * 0.45),
+    img_mean=tuple(round(x) for x in data_preprocessor['mean']),
+)
+
+train_pipeline = [
+    dict(type='mmcls.LoadImageFromFile'),
+    dict(type='mmcls.RandomResizedCrop', scale=224),
+    dict(type='mmcls.RandomFlip', prob=0.5, direction='horizontal'),
+    dict(type='mmcls.ColorJitter', brightness=0.1254, saturation=0.5),
+    dict(type='mmcls.PackClsInputs'),
+]
+
+test_pipeline = [
+    dict(type='mmcls.LoadImageFromFile'),
+    dict(
+        type='mmcls.ResizeEdge',
+        scale=256,
+        edge='short',
+        backend='pillow',
+        interpolation='bilinear'),
+    dict(type='mmcls.CenterCrop', crop_size=224),
+    dict(type='mmcls.PackClsInputs')
+]
+
+train_dataloader = dict(
+    batch_size=64,
+    num_workers=16,
+    dataset=dict(
+        type=dataset_type,
+        data_root='data/imagenet',
+        ann_file='meta/train.txt',
+        data_prefix='train',
+        pipeline=train_pipeline),
+    sampler=dict(type='mmcls.RepeatAugSampler', shuffle=True),
+    persistent_workers=True,
+)
+
+val_dataloader = dict(
+    batch_size=64,
+    num_workers=16,
+    dataset=dict(
+        type=dataset_type,
+        data_root='data/imagenet',
+        ann_file='meta/val.txt',
+        data_prefix='val',
+        pipeline=test_pipeline),
+    sampler=dict(type='mmcls.DefaultSampler', shuffle=False),
+    persistent_workers=True,
+)
+val_evaluator = dict(type='mmcls.Accuracy', topk=(1, 5))
+
+# If you want standard test, please manually configure the test dataset
+test_dataloader = val_dataloader
+test_evaluator = val_evaluator
+
+# optimizer
+optim_wrapper = dict(
+    optimizer=dict(
+        type='SGD', lr=0.8, momentum=0.9, weight_decay=0.00001, nesterov=True),
+    paramwise_cfg=dict(bias_decay_mult=0., norm_decay_mult=0.))
+
+# learning policy
+max_epochs = 360
+param_scheduler = [
+    dict(
+        type='LinearLR', start_factor=0.001, by_epoch=False, begin=0,
+        end=3125),
+    dict(
+        type='CosineAnnealingLR',
+        T_max=max_epochs,
+        eta_min=0,
+        by_epoch=True,
+        begin=0,
+        end=max_epochs,
+        convert_to_iter_based=True)
+]
+
+# train, val, test setting
+train_cfg = dict(by_epoch=True, max_epochs=max_epochs, val_interval=1)
+val_cfg = dict(type='mmrazor.SubnetValLoop', calibrate_sample_num=4096)
+test_cfg = dict(type='mmrazor.SubnetValLoop', calibrate_sample_num=4096)

configs/nas/mmcls/autoslim/autoslim_mbv2_1.5x_slimmable_subnet_8xb256_in1k.py

@@ -43,6 +43,4 @@
     broadcast_buffers=False,
     find_unused_parameters=True)
 
-optim_wrapper = dict(accumulative_counts=3)
-
 val_cfg = dict(type='mmrazor.SlimmableValLoop')

configs/nas/mmcls/autoslim/autoslim_mbv2_1.5x_supernet_8xb256_in1k.py

@@ -21,12 +21,12 @@
 )
 
 # !autoslim algorithm config
-num_samples = 2
+num_random_samples = 2
 model = dict(
     _delete_=True,
     _scope_='mmrazor',
     type='AutoSlim',
-    num_samples=num_samples,
+    num_random_samples=num_random_samples,
     architecture=supernet,
     data_preprocessor=data_preprocessor,
     distiller=dict(
@@ -59,8 +59,6 @@
     broadcast_buffers=False,
     find_unused_parameters=False)
 
-optim_wrapper = dict(accumulative_counts=num_samples + 2)
-
 # learning policy
 max_epochs = 50
 param_scheduler = dict(end=max_epochs)

configs/nas/mmcls/bignas/ATTENTIVE_SUBNET_A0.yaml

@@ -0,0 +1,64 @@
+backbone.first_channels:
+  chosen: 16
+backbone.last_channels:
+  chosen: 1792
+backbone.layers.1.kernel_size:
+  chosen: 3
+backbone.layers.1.expand_ratio:
+  chosen: 1
+backbone.layers.1.depth:
+  chosen: 1
+backbone.layers.1.out_channels:
+  chosen: 16
+backbone.layers.2.kernel_size:
+  chosen: 3
+backbone.layers.2.expand_ratio:
+  chosen: 4
+backbone.layers.2.depth:
+  chosen: 3
+backbone.layers.2.out_channels:
+  chosen: 24
+backbone.layers.3.kernel_size:
+  chosen: 3
+backbone.layers.3.expand_ratio:
+  chosen: 4
+backbone.layers.3.depth:
+  chosen: 3
+backbone.layers.3.out_channels:
+  chosen: 32
+backbone.layers.4.kernel_size:
+  chosen: 3
+backbone.layers.4.expand_ratio:
+  chosen: 4
+backbone.layers.4.depth:
+  chosen: 3
+backbone.layers.4.out_channels:
+  chosen: 64
+backbone.layers.5.kernel_size:
+  chosen: 3
+backbone.layers.5.expand_ratio:
+  chosen: 4
+backbone.layers.5.depth:
+  chosen: 3
+backbone.layers.5.out_channels:
+  chosen: 112
+backbone.layers.6.kernel_size:
+  chosen: 3
+backbone.layers.6.expand_ratio:
+  chosen: 6
+backbone.layers.6.depth:
+  chosen: 3
+backbone.layers.6.out_channels:
+  chosen: 192
+backbone.layers.7.kernel_size:
+  chosen: 3
+backbone.layers.7.expand_ratio:
+  chosen: 6
+backbone.layers.7.depth:
+  chosen: 1
+backbone.layers.7.out_channels:
+  chosen: 216
+input_shape:
+  chosen:
+  - 192
+  - 192

configs/nas/mmcls/bignas/ATTENTIVE_SUBNET_A1.yaml

@@ -0,0 +1,64 @@
+backbone.first_channels:
+  chosen: 16
+backbone.last_channels:
+  chosen: 1984
+backbone.layers.1.kernel_size:
+  chosen: 3
+backbone.layers.1.expand_ratio:
+  chosen: 1
+backbone.layers.1.depth:
+  chosen: 1
+backbone.layers.1.out_channels:
+  chosen: 16
+backbone.layers.2.kernel_size:
+  chosen: 3
+backbone.layers.2.expand_ratio:
+  chosen: 4
+backbone.layers.2.depth:
+  chosen: 3
+backbone.layers.2.out_channels:
+  chosen: 24
+backbone.layers.3.kernel_size:
+  chosen: 3
+backbone.layers.3.expand_ratio:
+  chosen: 4
+backbone.layers.3.depth:
+  chosen: 3
+backbone.layers.3.out_channels:
+  chosen: 32
+backbone.layers.4.kernel_size:
+  chosen: 5
+backbone.layers.4.expand_ratio:
+  chosen: 4
+backbone.layers.4.depth:
+  chosen: 3
+backbone.layers.4.out_channels:
+  chosen: 64
+backbone.layers.5.kernel_size:
+  chosen: 3
+backbone.layers.5.expand_ratio:
+  chosen: 4
+backbone.layers.5.depth:
+  chosen: 3
+backbone.layers.5.out_channels:
+  chosen: 112
+backbone.layers.6.kernel_size:
+  chosen: 5
+backbone.layers.6.expand_ratio:
+  chosen: 6
+backbone.layers.6.depth:
+  chosen: 3
+backbone.layers.6.out_channels:
+  chosen: 192
+backbone.layers.7.kernel_size:
+  chosen: 3
+backbone.layers.7.expand_ratio:
+  chosen: 6
+backbone.layers.7.depth:
+  chosen: 1
+backbone.layers.7.out_channels:
+  chosen: 216
+input_shape:
+  chosen:
+  - 224
+  - 224

configs/nas/mmcls/bignas/ATTENTIVE_SUBNET_A2.yaml

@@ -0,0 +1,64 @@
+backbone.first_channels:
+  chosen: 16
+backbone.last_channels:
+  chosen: 1984
+backbone.layers.1.kernel_size:
+  chosen: 3
+backbone.layers.1.expand_ratio:
+  chosen: 1
+backbone.layers.1.depth:
+  chosen: 1
+backbone.layers.1.out_channels:
+  chosen: 16
+backbone.layers.2.kernel_size:
+  chosen: 3
+backbone.layers.2.expand_ratio:
+  chosen: 4
+backbone.layers.2.depth:
+  chosen: 3
+backbone.layers.2.out_channels:
+  chosen: 24
+backbone.layers.3.kernel_size:
+  chosen: 3
+backbone.layers.3.expand_ratio:
+  chosen: 5
+backbone.layers.3.depth:
+  chosen: 3
+backbone.layers.3.out_channels:
+  chosen: 32
+backbone.layers.4.kernel_size:
+  chosen: 3
+backbone.layers.4.expand_ratio:
+  chosen: 4
+backbone.layers.4.depth:
+  chosen: 3
+backbone.layers.4.out_channels:
+  chosen: 64
+backbone.layers.5.kernel_size:
+  chosen: 3
+backbone.layers.5.expand_ratio:
+  chosen: 4
+backbone.layers.5.depth:
+  chosen: 3
+backbone.layers.5.out_channels:
+  chosen: 112
+backbone.layers.6.kernel_size:
+  chosen: 5
+backbone.layers.6.expand_ratio:
+  chosen: 6
+backbone.layers.6.depth:
+  chosen: 4
+backbone.layers.6.out_channels:
+  chosen: 200
+backbone.layers.7.kernel_size:
+  chosen: 3
+backbone.layers.7.expand_ratio:
+  chosen: 6
+backbone.layers.7.depth:
+  chosen: 1
+backbone.layers.7.out_channels:
+  chosen: 224
+input_shape:
+  chosen:
+  - 224
+  - 224