[Fix] fix a bug about validation of ava dataset (#527)

* fix a bug about validation of ava dataset * update changelog * fix lint * rename ava metrics & update eval hook * use f-string
open-mmlab · Jan 19, 2021 · 910d2fb · 910d2fb
1 parent 030f947
commit 910d2fb
Show file tree

Hide file tree

Showing 12 changed files with 24 additions and 15 deletions.
diff --git a/configs/detection/ava/slowfast_context_kinetics_pretrained_r50_4x16x1_20e_ava_rgb.py b/configs/detection/ava/slowfast_context_kinetics_pretrained_r50_4x16x1_20e_ava_rgb.py
@@ -159,7 +159,7 @@
 total_epochs = 20
 checkpoint_config = dict(interval=1)
 workflow = [('train', 1)]
-evaluation = dict(interval=1)
+evaluation = dict(interval=1, key_indicator='mAP@0.5IOU')
 log_config = dict(
     interval=20, hooks=[
         dict(type='TextLoggerHook'),

diff --git a/configs/detection/ava/slowfast_kinetics_pretrained_r50_4x16x1_20e_ava_rgb.py b/configs/detection/ava/slowfast_kinetics_pretrained_r50_4x16x1_20e_ava_rgb.py
@@ -158,7 +158,7 @@
 total_epochs = 20
 checkpoint_config = dict(interval=1)
 workflow = [('train', 1)]
-evaluation = dict(interval=1)
+evaluation = dict(interval=1, key_indicator='mAP@0.5IOU')
 log_config = dict(
     interval=20, hooks=[
         dict(type='TextLoggerHook'),

diff --git a/configs/detection/ava/slowfast_kinetics_pretrained_r50_8x8x1_20e_ava_rgb.py b/configs/detection/ava/slowfast_kinetics_pretrained_r50_8x8x1_20e_ava_rgb.py
@@ -158,7 +158,7 @@
 total_epochs = 20
 checkpoint_config = dict(interval=1)
 workflow = [('train', 1)]
-evaluation = dict(interval=1)
+evaluation = dict(interval=1, key_indicator='mAP@0.5IOU')
 log_config = dict(
     interval=20, hooks=[
         dict(type='TextLoggerHook'),

diff --git a/configs/detection/ava/slowonly_kinetics_pretrained_r101_8x8x1_20e_ava_rgb.py b/configs/detection/ava/slowonly_kinetics_pretrained_r101_8x8x1_20e_ava_rgb.py
@@ -143,7 +143,7 @@
 total_epochs = 20
 checkpoint_config = dict(interval=1)
 workflow = [('train', 1)]
-evaluation = dict(interval=1)
+evaluation = dict(interval=1, key_indicator='mAP@0.5IOU')
 log_config = dict(
     interval=20, hooks=[
         dict(type='TextLoggerHook'),

diff --git a/configs/detection/ava/slowonly_kinetics_pretrained_r50_4x16x1_20e_ava_rgb.py b/configs/detection/ava/slowonly_kinetics_pretrained_r50_4x16x1_20e_ava_rgb.py
@@ -142,7 +142,7 @@
 total_epochs = 20
 checkpoint_config = dict(interval=1)
 workflow = [('train', 1)]
-evaluation = dict(interval=1)
+evaluation = dict(interval=1, key_indicator='mAP@0.5IOU')
 log_config = dict(
     interval=20, hooks=[
         dict(type='TextLoggerHook'),

diff --git a/configs/detection/ava/slowonly_omnisource_pretrained_r101_8x8x1_20e_ava_rgb.py b/configs/detection/ava/slowonly_omnisource_pretrained_r101_8x8x1_20e_ava_rgb.py
@@ -142,7 +142,7 @@
 total_epochs = 20
 checkpoint_config = dict(interval=1)
 workflow = [('train', 1)]
-evaluation = dict(interval=1)
+evaluation = dict(interval=1, key_indicator='mAP@0.5IOU')
 log_config = dict(
     interval=20, hooks=[
         dict(type='TextLoggerHook'),

diff --git a/configs/detection/ava/slowonly_omnisource_pretrained_r50_4x16x1_20e_ava_rgb.py b/configs/detection/ava/slowonly_omnisource_pretrained_r50_4x16x1_20e_ava_rgb.py
@@ -143,7 +143,7 @@
 total_epochs = 20
 checkpoint_config = dict(interval=1)
 workflow = [('train', 1)]
-evaluation = dict(interval=1)
+evaluation = dict(interval=1, key_indicator='mAP@0.5IOU')
 log_config = dict(
     interval=20, hooks=[
         dict(type='TextLoggerHook'),

diff --git a/docs/changelog.md b/docs/changelog.md
@@ -2,11 +2,21 @@
 
 ### master
 
+**Highlights**
+
+**New Features**
+
 **Improvements**
 
 - Support setting `max_testing_views` for extremely large models to save GPU memory used ([#511](https://github.com/open-mmlab/mmaction2/pull/511))
 - Add repo citation ([#545](https://github.com/open-mmlab/mmaction2/pull/545))
 
+**Bug and Typo Fixes**
+
+- Fix a bug about ava dataset validation ([#527](https://github.com/open-mmlab/mmaction2/pull/527).
+
+**ModelZoo**
+
 ### 0.10.0 (31/12/2020)
 
 **Highlights**

diff --git a/mmaction/core/evaluation/ava_evaluation/object_detection_evaluation.py b/mmaction/core/evaluation/ava_evaluation/object_detection_evaluation.py
@@ -304,7 +304,7 @@ def evaluate(self):
             mean_corloc,
         ) = self._evaluation.evaluate()
 
-        metric = 'Precision/mAP@{}IOU'.format(self._matching_iou_threshold)
+        metric = f'mAP@{self._matching_iou_threshold}IOU'
         pascal_metrics = {self._metric_prefix + metric: mean_ap}
         if self._evaluate_corlocs:
             pascal_metrics[self._metric_prefix +
@@ -353,7 +353,6 @@ def __init__(self, categories, matching_iou_threshold=0.5):
             categories,
             matching_iou_threshold=matching_iou_threshold,
             evaluate_corlocs=False,
-            metric_prefix='PascalBoxes',
             use_weighted_mean_ap=False,
         )
 

diff --git a/mmaction/core/evaluation/eval_hooks.py b/mmaction/core/evaluation/eval_hooks.py
@@ -35,7 +35,9 @@ class EpochEvalHook(Hook):
             ``mean_average_precision``, ``mmit_mean_average_precision``
             for action recognition dataset (RawframeDataset and VideoDataset).
             ``AR@AN``, ``auc`` for action localization dataset
-            (ActivityNetDataset). Default: `top1_acc`.
+            (ActivityNetDataset). ``Recall@0.5@100``, ``AR@100``,
+            ``mAP@0.5IOU`` for spatio-temporal action detection dataset
+            (AVADataset). Default: `top1_acc`.
         rule (str | None): Comparison rule for best score. Options are None,
             'greater' and 'less'. If set to None, it will infer a reasonable
             rule. Default: 'None'.
@@ -45,7 +47,7 @@ class EpochEvalHook(Hook):
 
     rule_map = {'greater': lambda x, y: x > y, 'less': lambda x, y: x < y}
     init_value_map = {'greater': -inf, 'less': inf}
-    greater_keys = ['acc', 'top', 'AR@', 'auc', 'precision']
+    greater_keys = ['acc', 'top', 'AR@', 'auc', 'precision', 'mAP@', 'Recall@']
     less_keys = ['loss']
 
     def __init__(self,

diff --git a/tests/test_accuracy.py b/tests/test_accuracy.py
@@ -83,8 +83,7 @@ def test_ava_detection():
 
     # eval bbox
     detection = ava_eval(result_path, 'bbox', label_map, gt_path, None)
-    assert_array_almost_equal(detection['PascalBoxes_Precision/mAP@0.5IOU'],
-                              0.09385522)
+    assert_array_almost_equal(detection['mAP@0.5IOU'], 0.09385522)
 
 
 def test_confusion_matrix():

diff --git a/tests/test_data/test_ava_dataset.py b/tests/test_data/test_ava_dataset.py
@@ -172,5 +172,4 @@ def test_ava_evaluate(self):
         res = ava_dataset.evaluate(fake_result)
         assert_array_almost_equal(res['Recall@0.5@100'], 0.33333333)
         assert_array_almost_equal(res['AR@100'], 0.15833333)
-        assert_array_almost_equal(res['PascalBoxes_Precision/mAP@0.5IOU'],
-                                  0.027777778)
+        assert_array_almost_equal(res['mAP@0.5IOU'], 0.027777778)