[Feature] Support evaluating CocoMetric without ann_file. (open-mmla…

…b#1722)

configs/animal_2d_keypoint/topdown_heatmap/ap10k/td-hm_hrnet-w32_8xb64-210e_ap10k-256x256.py

@@ -27,7 +27,7 @@
 auto_scale_lr = dict(base_batch_size=512)
 
 # hooks
-default_hooks = dict(checkpoint=dict(save_best='coco/AP', rule='greater'))
+default_hooks = dict(checkpoint=dict(save_best='ap10k/AP', rule='greater'))
 
 # codec settings
 codec = dict(

configs/animal_2d_keypoint/topdown_heatmap/ap10k/td-hm_hrnet-w48_8xb64-210e_ap10k-256x256.py

@@ -27,7 +27,7 @@
 auto_scale_lr = dict(base_batch_size=512)
 
 # hooks
-default_hooks = dict(checkpoint=dict(save_best='coco/AP', rule='greater'))
+default_hooks = dict(checkpoint=dict(save_best='ap10k/AP', rule='greater'))
 
 # codec settings
 codec = dict(

configs/animal_2d_keypoint/topdown_heatmap/ap10k/td-hm_res101_8xb64-210e_ap10k-256x256.py

@@ -27,7 +27,7 @@
 auto_scale_lr = dict(base_batch_size=512)
 
 # hooks
-default_hooks = dict(checkpoint=dict(save_best='coco/AP', rule='greater'))
+default_hooks = dict(checkpoint=dict(save_best='ap10k/AP', rule='greater'))
 
 # codec settings
 codec = dict(

configs/animal_2d_keypoint/topdown_heatmap/ap10k/td-hm_res50_8xb64-210e_ap10k-256x256.py

@@ -27,7 +27,7 @@
 auto_scale_lr = dict(base_batch_size=512)
 
 # hooks
-default_hooks = dict(checkpoint=dict(save_best='coco/AP', rule='greater'))
+default_hooks = dict(checkpoint=dict(save_best='ap10k/AP', rule='greater'))
 
 # codec settings
 codec = dict(

mmpose/datasets/datasets/base/base_coco_style_dataset.py

@@ -1,4 +1,5 @@
 # Copyright (c) OpenMMLab. All rights reserved.
+import copy
 import os.path as osp
 from copy import deepcopy
 from itertools import filterfalse, groupby
@@ -184,6 +185,10 @@ def _load_annotations(self):
         check_file_exist(self.ann_file)
 
         coco = COCO(self.ann_file)
+        # set the metainfo about categories, which is a list of dict
+        # and each dict contains the 'id', 'name', etc. about this category
+        self._metainfo['CLASSES'] = coco.loadCats(coco.getCatIds())
+
         data_list = []
 
         for img_id in coco.getImgIds():
@@ -259,8 +264,14 @@ def parse_data_info(self, raw_data_info: dict) -> Optional[dict]:
             'iscrowd': ann.get('iscrowd', 0),
             'segmentation': ann.get('segmentation', None),
             'id': ann['id'],
+            # store the raw annotation of the instance
+            # it is useful for evaluation without providing ann_file
+            'raw_ann_info': copy.deepcopy(ann),
         }
 
+        if 'crowdIndex' in img:
+            data_info['crowdIndex'] = img['crowdIndex']
+
         return data_info
 
     @staticmethod
@@ -347,6 +358,9 @@ def _load_detection_results(self) -> List[dict]:
 
         # load coco annotations to build image id-to-name index
         coco = COCO(self.ann_file)
+        # set the metainfo about categories, which is a list of dict
+        # and each dict contains the 'id', 'name', etc. about this category
+        self._metainfo['CLASSES'] = coco.loadCats(coco.getCatIds())
 
         num_keypoints = self.metainfo['num_keypoints']
         data_list = []

mmpose/datasets/transforms/formatting.py

@@ -49,6 +49,9 @@ class PackPoseInputs(BaseTransform):
 
         - ``img_path``: path to the image file
 
+        - ``crowd_index`` (optional): measure the crowding level of an image,
+            defined in CrowdPose dataset
+
         - ``ori_shape``: original shape of the image as a tuple (h, w, c)
 
         - ``img_shape``: shape of the image input to the network as a tuple \
@@ -63,11 +66,14 @@ class PackPoseInputs(BaseTransform):
 
         - ``flip_indices``: the indices of each keypoint's symmetric keypoint
 
+        - ``raw_ann_info`` (optional): raw annotation of the instance(s)
+
     Args:
         meta_keys (Sequence[str], optional): Meta keys which will be stored in
             :obj: `PoseDataSample` as meta info. Defaults to ``('id',
-            'img_id', 'img_path', 'ori_shape', 'img_shape', 'input_size',
-            'flip', 'flip_direction', 'flip_indices)``
+            'img_id', 'img_path', 'crowd_index, 'ori_shape', 'img_shape',
+            'input_size', 'flip', 'flip_direction', 'flip_indices',
+            'raw_ann_info')``
     """
 
     # items in `instance_mapping_table` will be directly packed into
@@ -94,9 +100,9 @@ class PackPoseInputs(BaseTransform):
     }
 
     def __init__(self,
-                 meta_keys=('id', 'img_id', 'img_path', 'ori_shape',
-                            'img_shape', 'input_size', 'flip',
-                            'flip_direction', 'flip_indices'),
+                 meta_keys=('id', 'img_id', 'img_path', 'crowd_index',
+                            'ori_shape', 'img_shape', 'input_size', 'flip',
+                            'flip_direction', 'flip_indices', 'raw_ann_info'),
                  pack_transformed=False):
         self.meta_keys = meta_keys
         self.pack_transformed = pack_transformed

mmpose/evaluation/functional/nms.py

@@ -3,18 +3,20 @@
 # Original licence: Copyright (c) Microsoft, under the MIT License.
 # ------------------------------------------------------------------------------
 
+from typing import List, Optional
+
 import numpy as np
 
 
-def nms(dets, thr):
+def nms(dets: np.ndarray, thr: float) -> List[int]:
     """Greedily select boxes with high confidence and overlap <= thr.
 
     Args:
-        dets: [[x1, y1, x2, y2, score]].
-        thr: Retain overlap < thr.
+        dets (np.ndarray): [[x1, y1, x2, y2, score]].
+        thr (float): Retain overlap < thr.
 
     Returns:
-         list: Indexes to keep.
+        list: Indexes to keep.
     """
     if len(dets) == 0:
         return []
@@ -48,19 +50,38 @@ def nms(dets, thr):
     return keep
 
 
-def oks_iou(g, d, a_g, a_d, sigmas=None, vis_thr=None):
+def oks_iou(g: np.ndarray,
+            d: np.ndarray,
+            a_g: float,
+            a_d: np.ndarray,
+            sigmas: Optional[np.ndarray] = None,
+            vis_thr: Optional[float] = None) -> np.ndarray:
     """Calculate oks ious.
 
+    Note:
+
+        - number of keypoints: K
+        - number of instances: N
+
     Args:
-        g: Ground truth keypoints.
-        d: Detected keypoints.
-        a_g: Area of the ground truth object.
-        a_d: Area of the detected object.
-        sigmas: standard deviation of keypoint labelling.
-        vis_thr: threshold of the keypoint visibility.
+        g (np.ndarray): The instance to calculate OKS IOU with other
+            instances. Containing the keypoints coordinates. Shape: (K*3, )
+        d (np.ndarray): The rest instances. Containing the keypoints
+            coordinates. Shape: (N, K*3)
+        a_g (float): Area of the ground truth object.
+        a_d (np.ndarray): Area of the detected object. Shape: (N, )
+        sigmas (np.ndarray, optional): Keypoint labelling uncertainty.
+            Please refer to `COCO keypoint evaluation
+            <https://cocodataset.org/#keypoints-eval>`__ for more details.
+            If not given, use the sigmas on COCO dataset.
+            If specified, shape: (K, ). Defaults to ``None``
+        vis_thr(float, optional): Threshold of the keypoint visibility.
+            If specified, will calculate OKS based on those keypoints whose
+            visibility higher than vis_thr. If not given, calculate the OKS
+            based on all keypoints. Defaults to ``None``
 
     Returns:
-        list: The oks ious.
+        np.ndarray: The oks ious.
     """
     if sigmas is None:
         sigmas = np.array([
@@ -86,15 +107,26 @@ def oks_iou(g, d, a_g, a_d, sigmas=None, vis_thr=None):
     return ious
 
 
-def oks_nms(kpts_db, thr, sigmas=None, vis_thr=None, score_per_joint=False):
+def oks_nms(kpts_db: List[dict],
+            thr: float,
+            sigmas: Optional[np.ndarray] = None,
+            vis_thr: Optional[float] = None,
+            score_per_joint: bool = False):
     """OKS NMS implementations.
 
     Args:
-        kpts_db: keypoints.
-        thr: Retain overlap < thr.
-        sigmas: standard deviation of keypoint labelling.
-        vis_thr: threshold of the keypoint visibility.
-        score_per_joint: the input scores (in kpts_db) are per joint scores
+        kpts_db (List[dict]): The keypoints results of the same image.
+        thr (float): The threshold of NMS. Will retain oks overlap < thr.
+        sigmas (np.ndarray, optional): Keypoint labelling uncertainty.
+            Please refer to `COCO keypoint evaluation
+            <https://cocodataset.org/#keypoints-eval>`__ for more details.
+            If not given, use the sigmas on COCO dataset. Defaults to ``None``
+        vis_thr(float, optional): Threshold of the keypoint visibility.
+            If specified, will calculate OKS based on those keypoints whose
+            visibility higher than vis_thr. If not given, calculate the OKS
+            based on all keypoints. Defaults to ``None``
+        score_per_joint(bool): Whether the input scores (in kpts_db) are
+            per-joint scores. Defaults to ``False``
 
     Returns:
         np.ndarray: indexes to keep.
@@ -128,14 +160,18 @@ def oks_nms(kpts_db, thr, sigmas=None, vis_thr=None, score_per_joint=False):
     return keep
 
 
-def _rescore(overlap, scores, thr, type='gaussian'):
+def _rescore(overlap: np.ndarray,
+             scores: np.ndarray,
+             thr: float,
+             type: str = 'gaussian'):
     """Rescoring mechanism gaussian or linear.
 
     Args:
-        overlap: calculated ious
-        scores: target scores.
-        thr: retain oks overlap < thr.
-        type: 'gaussian' or 'linear'
+        overlap (np.ndarray): The calculated oks ious.
+        scores (np.ndarray): target scores.
+        thr (float): retain oks overlap < thr.
+        type (str): The rescoring type. Could be 'gaussian' or 'linear'.
+            Defaults to ``'gaussian'``
 
     Returns:
         np.ndarray: indexes to keep
@@ -152,20 +188,28 @@ def _rescore(overlap, scores, thr, type='gaussian'):
     return scores
 
 
-def soft_oks_nms(kpts_db,
-                 thr,
-                 max_dets=20,
-                 sigmas=None,
-                 vis_thr=None,
-                 score_per_joint=False):
+def soft_oks_nms(kpts_db: List[dict],
+                 thr: float,
+                 max_dets: int = 20,
+                 sigmas: Optional[np.ndarray] = None,
+                 vis_thr: Optional[float] = None,
+                 score_per_joint: bool = False):
     """Soft OKS NMS implementations.
 
     Args:
-        kpts_db
-        thr: retain oks overlap < thr.
-        max_dets: max number of detections to keep.
-        sigmas: Keypoint labelling uncertainty.
-        score_per_joint: the input scores (in kpts_db) are per joint scores
+        kpts_db (List[dict]): The keypoints results of the same image.
+        thr (float): The threshold of NMS. Will retain oks overlap < thr.
+        max_dets (int): Maximum number of detections to keep. Defaults to 20
+        sigmas (np.ndarray, optional): Keypoint labelling uncertainty.
+            Please refer to `COCO keypoint evaluation
+            <https://cocodataset.org/#keypoints-eval>`__ for more details.
+            If not given, use the sigmas on COCO dataset. Defaults to ``None``
+        vis_thr(float, optional): Threshold of the keypoint visibility.
+            If specified, will calculate OKS based on those keypoints whose
+            visibility higher than vis_thr. If not given, calculate the OKS
+            based on all keypoints. Defaults to ``None``
+        score_per_joint(bool): Whether the input scores (in kpts_db) are
+            per-joint scores. Defaults to ``False``
 
     Returns:
         np.ndarray: indexes to keep.

mmpose/evaluation/metrics/__init__.py

@@ -1,11 +1,12 @@
 # Copyright (c) OpenMMLab. All rights reserved.
-from .coco_metric import CocoMetric
+from .coco_metric import AP10KCocoMetric, CocoMetric
 from .coco_wholebody_metric import CocoWholeBodyMetric
 from .keypoint_2d_metrics import (AUC, EPE, NME, JhmdbPCKAccuracy,
                                   MpiiPCKAccuracy, PCKAccuracy)
 from .posetrack18_metric import PoseTrack18Metric
 
 __all__ = [
-    'CocoMetric', 'PCKAccuracy', 'MpiiPCKAccuracy', 'JhmdbPCKAccuracy', 'AUC',
-    'EPE', 'NME', 'PoseTrack18Metric', 'CocoWholeBodyMetric'
+    'CocoMetric', 'AP10KCocoMetric', 'PCKAccuracy', 'MpiiPCKAccuracy',
+    'JhmdbPCKAccuracy', 'AUC', 'EPE', 'NME', 'PoseTrack18Metric',
+    'CocoWholeBodyMetric'
 ]