Merge a5da393 into 26f3688

open-mmlab · Sep 22, 2023 · 5e9408a · 5e9408a
2 parents 26f3688 + a5da393
commit 5e9408a
Show file tree

Hide file tree

Showing 12 changed files with 677 additions and 2 deletions.
diff --git a/configs/_base_/datasets/300wlp.py b/configs/_base_/datasets/300wlp.py
@@ -0,0 +1,86 @@
+dataset_info = dict(
+    dataset_name='300wlp',
+    paper_info=dict(
+        author='Xiangyu Zhu1, and Zhen Lei1 '
+        'and Xiaoming Liu2, and Hailin Shi1 '
+        'and Stan Z. Li1',
+        title='300 faces in-the-wild challenge: '
+        'Database and results',
+        container='Image and vision computing',
+        year='2016',
+        homepage='http://www.cbsr.ia.ac.cn/users/xiangyuzhu/'
+        'projects/3DDFA/main.htm',
+    ),
+    keypoint_info={
+        0: dict(name='kpt-0', id=0, color=[255, 0, 0], type='', swap=''),
+        1: dict(name='kpt-1', id=1, color=[255, 0, 0], type='', swap=''),
+        2: dict(name='kpt-2', id=2, color=[255, 0, 0], type='', swap=''),
+        3: dict(name='kpt-3', id=3, color=[255, 0, 0], type='', swap=''),
+        4: dict(name='kpt-4', id=4, color=[255, 0, 0], type='', swap=''),
+        5: dict(name='kpt-5', id=5, color=[255, 0, 0], type='', swap=''),
+        6: dict(name='kpt-6', id=6, color=[255, 0, 0], type='', swap=''),
+        7: dict(name='kpt-7', id=7, color=[255, 0, 0], type='', swap=''),
+        8: dict(name='kpt-8', id=8, color=[255, 0, 0], type='', swap=''),
+        9: dict(name='kpt-9', id=9, color=[255, 0, 0], type='', swap=''),
+        10: dict(name='kpt-10', id=10, color=[255, 0, 0], type='', swap=''),
+        11: dict(name='kpt-11', id=11, color=[255, 0, 0], type='', swap=''),
+        12: dict(name='kpt-12', id=12, color=[255, 0, 0], type='', swap=''),
+        13: dict(name='kpt-13', id=13, color=[255, 0, 0], type='', swap=''),
+        14: dict(name='kpt-14', id=14, color=[255, 0, 0], type='', swap=''),
+        15: dict(name='kpt-15', id=15, color=[255, 0, 0], type='', swap=''),
+        16: dict(name='kpt-16', id=16, color=[255, 0, 0], type='', swap=''),
+        17: dict(name='kpt-17', id=17, color=[255, 0, 0], type='', swap=''),
+        18: dict(name='kpt-18', id=18, color=[255, 0, 0], type='', swap=''),
+        19: dict(name='kpt-19', id=19, color=[255, 0, 0], type='', swap=''),
+        20: dict(name='kpt-20', id=20, color=[255, 0, 0], type='', swap=''),
+        21: dict(name='kpt-21', id=21, color=[255, 0, 0], type='', swap=''),
+        22: dict(name='kpt-22', id=22, color=[255, 0, 0], type='', swap=''),
+        23: dict(name='kpt-23', id=23, color=[255, 0, 0], type='', swap=''),
+        24: dict(name='kpt-24', id=24, color=[255, 0, 0], type='', swap=''),
+        25: dict(name='kpt-25', id=25, color=[255, 0, 0], type='', swap=''),
+        26: dict(name='kpt-26', id=26, color=[255, 0, 0], type='', swap=''),
+        27: dict(name='kpt-27', id=27, color=[255, 0, 0], type='', swap=''),
+        28: dict(name='kpt-28', id=28, color=[255, 0, 0], type='', swap=''),
+        29: dict(name='kpt-29', id=29, color=[255, 0, 0], type='', swap=''),
+        30: dict(name='kpt-30', id=30, color=[255, 0, 0], type='', swap=''),
+        31: dict(name='kpt-31', id=31, color=[255, 0, 0], type='', swap=''),
+        32: dict(name='kpt-32', id=32, color=[255, 0, 0], type='', swap=''),
+        33: dict(name='kpt-33', id=33, color=[255, 0, 0], type='', swap=''),
+        34: dict(name='kpt-34', id=34, color=[255, 0, 0], type='', swap=''),
+        35: dict(name='kpt-35', id=35, color=[255, 0, 0], type='', swap=''),
+        36: dict(name='kpt-36', id=36, color=[255, 0, 0], type='', swap=''),
+        37: dict(name='kpt-37', id=37, color=[255, 0, 0], type='', swap=''),
+        38: dict(name='kpt-38', id=38, color=[255, 0, 0], type='', swap=''),
+        39: dict(name='kpt-39', id=39, color=[255, 0, 0], type='', swap=''),
+        40: dict(name='kpt-40', id=40, color=[255, 0, 0], type='', swap=''),
+        41: dict(name='kpt-41', id=41, color=[255, 0, 0], type='', swap=''),
+        42: dict(name='kpt-42', id=42, color=[255, 0, 0], type='', swap=''),
+        43: dict(name='kpt-43', id=43, color=[255, 0, 0], type='', swap=''),
+        44: dict(name='kpt-44', id=44, color=[255, 0, 0], type='', swap=''),
+        45: dict(name='kpt-45', id=45, color=[255, 0, 0], type='', swap=''),
+        46: dict(name='kpt-46', id=46, color=[255, 0, 0], type='', swap=''),
+        47: dict(name='kpt-47', id=47, color=[255, 0, 0], type='', swap=''),
+        48: dict(name='kpt-48', id=48, color=[255, 0, 0], type='', swap=''),
+        49: dict(name='kpt-49', id=49, color=[255, 0, 0], type='', swap=''),
+        50: dict(name='kpt-50', id=50, color=[255, 0, 0], type='', swap=''),
+        51: dict(name='kpt-51', id=51, color=[255, 0, 0], type='', swap=''),
+        52: dict(name='kpt-52', id=52, color=[255, 0, 0], type='', swap=''),
+        53: dict(name='kpt-53', id=53, color=[255, 0, 0], type='', swap=''),
+        54: dict(name='kpt-54', id=54, color=[255, 0, 0], type='', swap=''),
+        55: dict(name='kpt-55', id=55, color=[255, 0, 0], type='', swap=''),
+        56: dict(name='kpt-56', id=56, color=[255, 0, 0], type='', swap=''),
+        57: dict(name='kpt-57', id=57, color=[255, 0, 0], type='', swap=''),
+        58: dict(name='kpt-58', id=58, color=[255, 0, 0], type='', swap=''),
+        59: dict(name='kpt-59', id=59, color=[255, 0, 0], type='', swap=''),
+        60: dict(name='kpt-60', id=60, color=[255, 0, 0], type='', swap=''),
+        61: dict(name='kpt-61', id=61, color=[255, 0, 0], type='', swap=''),
+        62: dict(name='kpt-62', id=62, color=[255, 0, 0], type='', swap=''),
+        63: dict(name='kpt-63', id=63, color=[255, 0, 0], type='', swap=''),
+        64: dict(name='kpt-64', id=64, color=[255, 0, 0], type='', swap=''),
+        65: dict(name='kpt-65', id=65, color=[255, 0, 0], type='', swap=''),
+        66: dict(name='kpt-66', id=66, color=[255, 0, 0], type='', swap=''),
+        67: dict(name='kpt-67', id=67, color=[255, 0, 0], type='', swap=''),
+    },
+    skeleton_info={},
+    joint_weights=[1.] * 68,
+    sigmas=[])
diff --git a/configs/face_2d_keypoint/topdown_heatmap/300wlp/hrnetv2_300wlp.md b/configs/face_2d_keypoint/topdown_heatmap/300wlp/hrnetv2_300wlp.md
@@ -0,0 +1,42 @@
+<!-- [ALGORITHM] -->
+
+<details>
+<summary align="right"><a href="https://ieeexplore.ieee.org/abstract/document/9052469/">HRNetv2 (TPAMI'2019)</a></summary>
+
+```bibtex
+@article{WangSCJDZLMTWLX19,
+  title={Deep High-Resolution Representation Learning for Visual Recognition},
+  author={Jingdong Wang and Ke Sun and Tianheng Cheng and
+          Borui Jiang and Chaorui Deng and Yang Zhao and Dong Liu and Yadong Mu and
+          Mingkui Tan and Xinggang Wang and Wenyu Liu and Bin Xiao},
+  journal={TPAMI},
+  year={2019}
+}
+```
+
+</details>
+
+<!-- [DATASET] -->
+
+<details>
+<summary align="right"><a href="http://www.cbsr.ia.ac.cn/users/xiangyuzhu/projects/3DDFA/main.htm">300WLP (IEEE'2017)</a></summary>
+
+```bibtex
+@article{zhu2017face,
+  title={Face alignment in full pose range: A 3d total solution},
+  author={Zhu, Xiangyu and Liu, Xiaoming and Lei, Zhen and Li, Stan Z},
+  journal={IEEE transactions on pattern analysis and machine intelligence},
+  year={2017},
+  publisher={IEEE}
+}
+```
+
+</details>
+
+Results on 300W-LP dataset
+
+The model is trained on 300W-LP train.
+
+| Arch                                               | Input Size | NME<sub>*full*</sub> | NME<sub>*test*</sub> |                        ckpt                        |                        log                         |
+| :------------------------------------------------- | :--------: | :------------------: | :------------------: | :------------------------------------------------: | :------------------------------------------------: |
+| [pose_hrnetv2_w18](/configs/face_2d_keypoint/topdown_heatmap/300wlp/td-hm_hrnetv2-w18_8xb64-60e_300wlp-256x256.py) |  256x256   |        0.0413        |       0.04125        | [ckpt](https://download.openmmlab.com/mmpose/v1/face_2d_keypoint/topdown_heatmap/300wlp/hrnetv2_w18_300wlp_256x256-fb433d21_20230922.pth) | [log](https://download.openmmlab.com/mmpose/v1/face_2d_keypoint/topdown_heatmap/300wlp/hrnetv2_w18_300wlp_256x256-fb433d21_20230922.json) |
diff --git a/configs/face_2d_keypoint/topdown_heatmap/300wlp/hrnetv2_300wlp.yml b/configs/face_2d_keypoint/topdown_heatmap/300wlp/hrnetv2_300wlp.yml
@@ -0,0 +1,20 @@
+Collections:
+- Name: HRNetv2
+  Paper:
+    Title: Deep High-Resolution Representation Learning for Visual Recognition
+    URL: https://ieeexplore.ieee.org/abstract/document/9052469/
+  README: https://github.com/open-mmlab/mmpose/blob/main/docs/src/papers/backbones/hrnetv2.md
+Models:
+- Config: configs/face_2d_keypoint/topdown_heatmap/300wlp/td-hm_hrnetv2-w18_8xb64-60e_300wlp-256x256.py
+  In Collection: HRNetv2
+  Metadata:
+    Architecture:
+    - HRNetv2
+    Training Data: 300W-LP
+  Name: td-hm_hrnetv2-w18_8xb64-60e_300wlp-256x256
+  Results:
+  - Dataset: 300W-LP
+    Metrics:
+      NME full: 0.0413
+    Task: Face 2D Keypoint
+  Weights: https://download.openmmlab.com/mmpose/v1/face_2d_keypoint/topdown_heatmap/300wlp/hrnetv2_w18_300wlp_256x256-fb433d21_20230922.pth
diff --git a/...igs/face_2d_keypoint/topdown_heatmap/300wlp/td-hm_hrnetv2-w18_8xb64-60e_300wlp-256x256.py b/...igs/face_2d_keypoint/topdown_heatmap/300wlp/td-hm_hrnetv2-w18_8xb64-60e_300wlp-256x256.py
@@ -0,0 +1,160 @@
+_base_ = ['../../../_base_/default_runtime.py']
+
+# runtime
+train_cfg = dict(max_epochs=60, val_interval=1)
+
+# optimizer
+optim_wrapper = dict(optimizer=dict(
+    type='Adam',
+    lr=2e-3,
+))
+
+# learning policy
+param_scheduler = [
+    dict(
+        type='LinearLR', begin=0, end=500, start_factor=0.001,
+        by_epoch=False),  # warm-up
+    dict(
+        type='MultiStepLR',
+        begin=0,
+        end=60,
+        milestones=[40, 55],
+        gamma=0.1,
+        by_epoch=True)
+]
+
+# automatically scaling LR based on the actual training batch size
+auto_scale_lr = dict(base_batch_size=512)
+
+# hooks
+default_hooks = dict(checkpoint=dict(save_best='NME', rule='less', interval=1))
+
+# codec settings
+codec = dict(
+    type='MSRAHeatmap',
+    input_size=(256, 256),
+    heatmap_size=(64, 64),
+    sigma=1.5)
+
+# model settings
+model = dict(
+    type='TopdownPoseEstimator',
+    data_preprocessor=dict(
+        type='PoseDataPreprocessor',
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        bgr_to_rgb=True),
+    backbone=dict(
+        type='HRNet',
+        in_channels=3,
+        extra=dict(
+            stage1=dict(
+                num_modules=1,
+                num_branches=1,
+                block='BOTTLENECK',
+                num_blocks=(4, ),
+                num_channels=(64, )),
+            stage2=dict(
+                num_modules=1,
+                num_branches=2,
+                block='BASIC',
+                num_blocks=(4, 4),
+                num_channels=(18, 36)),
+            stage3=dict(
+                num_modules=4,
+                num_branches=3,
+                block='BASIC',
+                num_blocks=(4, 4, 4),
+                num_channels=(18, 36, 72)),
+            stage4=dict(
+                num_modules=3,
+                num_branches=4,
+                block='BASIC',
+                num_blocks=(4, 4, 4, 4),
+                num_channels=(18, 36, 72, 144),
+                multiscale_output=True),
+            upsample=dict(mode='bilinear', align_corners=False)),
+        init_cfg=dict(
+            type='Pretrained', checkpoint='open-mmlab://msra/hrnetv2_w18'),
+    ),
+    neck=dict(
+        type='FeatureMapProcessor',
+        concat=True,
+    ),
+    head=dict(
+        type='HeatmapHead',
+        in_channels=270,
+        out_channels=68,
+        deconv_out_channels=None,
+        conv_out_channels=(270, ),
+        conv_kernel_sizes=(1, ),
+        loss=dict(type='KeypointMSELoss', use_target_weight=True),
+        decoder=codec),
+    test_cfg=dict(
+        flip_test=True,
+        flip_mode='heatmap',
+        shift_heatmap=True,
+    ))
+
+# base dataset settings
+dataset_type = 'Face300WLPDataset'
+data_mode = 'topdown'
+data_root = 'data/300wlp/'
+# pipelines
+train_pipeline = [
+    dict(type='LoadImage'),
+    dict(type='GetBBoxCenterScale'),
+    dict(type='RandomFlip', direction='horizontal'),
+    dict(
+        type='RandomBBoxTransform',
+        shift_prob=0,
+        rotate_factor=60,
+        scale_factor=(0.75, 1.25)),
+    dict(type='TopdownAffine', input_size=codec['input_size']),
+    dict(type='GenerateTarget', encoder=codec),
+    dict(type='PackPoseInputs')
+]
+val_pipeline = [
+    dict(type='LoadImage'),
+    dict(type='GetBBoxCenterScale'),
+    dict(type='TopdownAffine', input_size=codec['input_size']),
+    dict(type='PackPoseInputs')
+]
+
+# data loaders
+train_dataloader = dict(
+    batch_size=2,
+    num_workers=2,
+    persistent_workers=True,
+    sampler=dict(type='DefaultSampler', shuffle=True),
+    dataset=dict(
+        type=dataset_type,
+        data_root=data_root,
+        data_mode=data_mode,
+        ann_file='annotations/face_landmarks_300wlp_train.json',
+        data_prefix=dict(img='train/'),
+        pipeline=train_pipeline,
+    ))
+val_dataloader = dict(
+    batch_size=32,
+    num_workers=2,
+    persistent_workers=True,
+    drop_last=False,
+    sampler=dict(type='DefaultSampler', shuffle=False, round_up=False),
+    dataset=dict(
+        type=dataset_type,
+        data_root=data_root,
+        data_mode=data_mode,
+        ann_file='annotations/face_landmarks_300wlp_valid.json',
+        data_prefix=dict(img='val/'),
+        test_mode=True,
+        pipeline=val_pipeline,
+    ))
+test_dataloader = val_dataloader
+
+# evaluators
+val_evaluator = dict(
+    type='NME',
+    norm_mode='keypoint_distance',
+)
+test_evaluator = val_evaluator
diff --git a/mmpose/datasets/datasets/face/__init__.py b/mmpose/datasets/datasets/face/__init__.py
@@ -3,10 +3,11 @@
 from .coco_wholebody_face_dataset import CocoWholeBodyFaceDataset
 from .cofw_dataset import COFWDataset
 from .face_300w_dataset import Face300WDataset
+from .face_300wlp_dataset import Face300WLPDataset
 from .lapa_dataset import LapaDataset
 from .wflw_dataset import WFLWDataset
 
 __all__ = [
     'Face300WDataset', 'WFLWDataset', 'AFLWDataset', 'COFWDataset',
-    'CocoWholeBodyFaceDataset', 'LapaDataset'
+    'CocoWholeBodyFaceDataset', 'LapaDataset', 'Face300WLPDataset'
 ]
diff --git a/mmpose/datasets/datasets/face/face_300wlp_dataset.py b/mmpose/datasets/datasets/face/face_300wlp_dataset.py
@@ -0,0 +1,55 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+
+from mmpose.registry import DATASETS
+from ..base import BaseCocoStyleDataset
+
+
+@DATASETS.register_module()
+class Face300WLPDataset(BaseCocoStyleDataset):
+    """300W dataset for face keypoint localization.
+
+    "300 faces In-the-wild challenge: Database and results",
+    Image and Vision Computing (IMAVIS) 2019.
+
+    The landmark annotations follow the 68 points mark-up. The definition
+    can be found in `https://ibug.doc.ic.ac.uk/resources/300-W/`.
+
+    Args:
+        ann_file (str): Annotation file path. Default: ''.
+        bbox_file (str, optional): Detection result file path. If
+            ``bbox_file`` is set, detected bboxes loaded from this file will
+            be used instead of ground-truth bboxes. This setting is only for
+            evaluation, i.e., ignored when ``test_mode`` is ``False``.
+            Default: ``None``.
+        data_mode (str): Specifies the mode of data samples: ``'topdown'`` or
+            ``'bottomup'``. In ``'topdown'`` mode, each data sample contains
+            one instance; while in ``'bottomup'`` mode, each data sample
+            contains all instances in a image. Default: ``'topdown'``
+        metainfo (dict, optional): Meta information for dataset, such as class
+            information. Default: ``None``.
+        data_root (str, optional): The root directory for ``data_prefix`` and
+            ``ann_file``. Default: ``None``.
+        data_prefix (dict, optional): Prefix for training data. Default:
+            ``dict(img=None, ann=None)``.
+        filter_cfg (dict, optional): Config for filter data. Default: `None`.
+        indices (int or Sequence[int], optional): Support using first few
+            data in annotation file to facilitate training/testing on a smaller
+            dataset. Default: ``None`` which means using all ``data_infos``.
+        serialize_data (bool, optional): Whether to hold memory using
+            serialized objects, when enabled, data loader workers can use
+            shared RAM from master process instead of making a copy.
+            Default: ``True``.
+        pipeline (list, optional): Processing pipeline. Default: [].
+        test_mode (bool, optional): ``test_mode=True`` means in test phase.
+            Default: ``False``.
+        lazy_init (bool, optional): Whether to load annotation during
+            instantiation. In some cases, such as visualization, only the meta
+            information of the dataset is needed, which is not necessary to
+            load annotation file. ``Basedataset`` can skip load annotations to
+            save time by set ``lazy_init=False``. Default: ``False``.
+        max_refetch (int, optional): If ``Basedataset.prepare_data`` get a
+            None img. The maximum extra number of cycles to get a valid
+            image. Default: 1000.
+    """
+
+    METAINFO: dict = dict(from_file='configs/_base_/datasets/300wlp.py')
diff --git a/projects/just_dance/configs/_base_ b/projects/just_dance/configs/_base_
@@ -1 +1 @@
-../../../configs/_base_
+../../../configs/_base_
diff --git a/tests/data/300wlp/AFW_134212_1_0.jpg b/tests/data/300wlp/AFW_134212_1_0.jpg
diff --git a/tests/data/300wlp/AFW_134212_2_0.jpg b/tests/data/300wlp/AFW_134212_2_0.jpg