[Squash] Refator ViT (from #295)

open-mmlab · Aug 5, 2021 · 18f62d5 · 18f62d5
1 parent a747458
commit 18f62d5
Show file tree

Hide file tree

Showing 13 changed files with 530 additions and 318 deletions.
diff --git a/configs/_base_/default_runtime.py b/configs/_base_/default_runtime.py
@@ -1,5 +1,5 @@
 # checkpoint saving
-checkpoint_config = dict(interval=1)
+checkpoint_config = dict(interval=10)
 # yapf:disable
 log_config = dict(
     interval=100,

diff --git a/configs/_base_/models/vit_base_patch16_224_finetune.py b/configs/_base_/models/vit_base_patch16_224_finetune.py
@@ -3,19 +3,25 @@
     type='ImageClassifier',
     backbone=dict(
         type='VisionTransformer',
-        num_layers=12,
-        embed_dim=768,
-        num_heads=12,
+        arch='b',
         img_size=224,
         patch_size=16,
         in_channels=3,
-        feedforward_channels=3072,
-        drop_rate=0.1),
+        drop_rate=0.1,
+        attn_drop_rate=0.,
+        hybrid_backbone=None,
+        init_cfg=[
+            dict(
+                type='Kaiming',
+                layer='Conv2d',
+                mode='fan_in',
+                nonlinearity='linear')
+        ]),
     neck=None,
     head=dict(
         type='VisionTransformerClsHead',
         num_classes=1000,
         in_channels=768,
-        loss=dict(type='CrossEntropyLoss', loss_weight=1.0),
+        loss=dict(type='LabelSmoothLoss', label_smooth_val=0.1),
         topk=(1, 5),
     ))
diff --git a/configs/_base_/models/vit_base_patch16_224_pretrain.py b/configs/_base_/models/vit_base_patch16_224_pretrain.py
@@ -3,15 +3,20 @@
     type='ImageClassifier',
     backbone=dict(
         type='VisionTransformer',
-        num_layers=12,
-        embed_dim=768,
-        num_heads=12,
+        arch='b',
         img_size=224,
         patch_size=16,
         in_channels=3,
-        feedforward_channels=3072,
         drop_rate=0.1,
-        attn_drop_rate=0.),
+        attn_drop_rate=0.,
+        hybrid_backbone=None,
+        init_cfg=[
+            dict(
+                type='Kaiming',
+                layer='Conv2d',
+                mode='fan_in',
+                nonlinearity='linear')
+        ]),
     neck=None,
     head=dict(
         type='VisionTransformerClsHead',

diff --git a/configs/_base_/models/vit_base_patch16_384_finetune.py b/configs/_base_/models/vit_base_patch16_384_finetune.py
@@ -3,19 +3,25 @@
     type='ImageClassifier',
     backbone=dict(
         type='VisionTransformer',
-        num_layers=12,
-        embed_dim=768,
-        num_heads=12,
+        arch='b',
         img_size=384,
         patch_size=16,
         in_channels=3,
-        feedforward_channels=3072,
-        drop_rate=0.1),
+        drop_rate=0.1,
+        attn_drop_rate=0.,
+        hybrid_backbone=None,
+        init_cfg=[
+            dict(
+                type='Kaiming',
+                layer='Conv2d',
+                mode='fan_in',
+                nonlinearity='linear')
+        ]),
     neck=None,
     head=dict(
         type='VisionTransformerClsHead',
         num_classes=1000,
         in_channels=768,
-        loss=dict(type='CrossEntropyLoss', loss_weight=1.0),
+        loss=dict(type='LabelSmoothLoss', label_smooth_val=0.1),
         topk=(1, 5),
     ))
diff --git a/configs/_base_/models/vit_base_patch32_384_finetune.py b/configs/_base_/models/vit_base_patch32_384_finetune.py
@@ -3,14 +3,20 @@
     type='ImageClassifier',
     backbone=dict(
         type='VisionTransformer',
-        num_layers=12,
-        embed_dim=768,
-        num_heads=12,
+        arch='b',
         img_size=384,
         patch_size=32,
         in_channels=3,
-        feedforward_channels=3072,
-        drop_rate=0.1),
+        drop_rate=0.1,
+        attn_drop_rate=0.,
+        hybrid_backbone=None,
+        init_cfg=[
+            dict(
+                type='Kaiming',
+                layer='Conv2d',
+                mode='fan_in',
+                nonlinearity='linear')
+        ]),
     neck=None,
     head=dict(
         type='VisionTransformerClsHead',

diff --git a/configs/_base_/models/vit_large_patch16_224_finetune.py b/configs/_base_/models/vit_large_patch16_224_finetune.py
@@ -3,14 +3,20 @@
     type='ImageClassifier',
     backbone=dict(
         type='VisionTransformer',
-        num_layers=24,
-        embed_dim=1024,
-        num_heads=16,
+        arch='l',
         img_size=224,
         patch_size=16,
         in_channels=3,
-        feedforward_channels=4096,
-        drop_rate=0.1),
+        drop_rate=0.1,
+        attn_drop_rate=0.,
+        hybrid_backbone=None,
+        init_cfg=[
+            dict(
+                type='Kaiming',
+                layer='Conv2d',
+                mode='fan_in',
+                nonlinearity='linear')
+        ]),
     neck=None,
     head=dict(
         type='VisionTransformerClsHead',

diff --git a/configs/_base_/models/vit_large_patch16_384_finetune.py b/configs/_base_/models/vit_large_patch16_384_finetune.py
@@ -3,14 +3,20 @@
     type='ImageClassifier',
     backbone=dict(
         type='VisionTransformer',
-        num_layers=24,
-        embed_dim=1024,
-        num_heads=16,
+        arch='l',
         img_size=384,
         patch_size=16,
         in_channels=3,
-        feedforward_channels=4096,
-        drop_rate=0.1),
+        drop_rate=0.1,
+        attn_drop_rate=0.,
+        hybrid_backbone=None,
+        init_cfg=[
+            dict(
+                type='Kaiming',
+                layer='Conv2d',
+                mode='fan_in',
+                nonlinearity='linear')
+        ]),
     neck=None,
     head=dict(
         type='VisionTransformerClsHead',

diff --git a/configs/_base_/models/vit_large_patch32_384_finetune.py b/configs/_base_/models/vit_large_patch32_384_finetune.py
@@ -3,14 +3,20 @@
     type='ImageClassifier',
     backbone=dict(
         type='VisionTransformer',
-        num_layers=24,
-        embed_dim=1024,
-        num_heads=16,
+        arch='l',
         img_size=384,
         patch_size=32,
         in_channels=3,
-        feedforward_channels=4096,
-        drop_rate=0.1),
+        drop_rate=0.1,
+        attn_drop_rate=0.,
+        hybrid_backbone=None,
+        init_cfg=[
+            dict(
+                type='Kaiming',
+                layer='Conv2d',
+                mode='fan_in',
+                nonlinearity='linear')
+        ]),
     neck=None,
     head=dict(
         type='VisionTransformerClsHead',