Fix bug in detection mAP evaluator. (#8778)

* Fix mAP evaluator bug.

* Fix bug in detection mAP evaluator.

* Fix unit testing.

* Support to set background label index in detection mAP op.

paddle/fluid/operators/detection_map_op.cc

@@ -142,7 +142,15 @@ class DetectionMAPOpMaker : public framework::OpProtoAndCheckerMaker {
     AddOutput("MAP",
               "(Tensor) A tensor with shape [1], store the mAP evaluate "
               "result of the detection.");
-
+    AddAttr<int>("class_num",
+                 "(int) "
+                 "The class number.");
+    AddAttr<int>(
+        "background_label",
+        "(int, defalut: 0) "
+        "The index of background label, the background label will be ignored. "
+        "If set to -1, then all categories will be considered.")
+        .SetDefault(0);
     AddAttr<float>(
         "overlap_threshold",
         "(float) "

paddle/fluid/operators/detection_map_op.h

@@ -69,6 +69,7 @@ class DetectionMAPOpKernel : public framework::OpKernel<T> {
     float overlap_threshold = ctx.Attr<float>("overlap_threshold");
     float evaluate_difficult = ctx.Attr<bool>("evaluate_difficult");
     auto ap_type = GetAPType(ctx.Attr<std::string>("ap_type"));
+    int class_num = ctx.Attr<int>("class_num");
 
     auto label_lod = in_label->lod();
     auto detect_lod = in_detect->lod();
@@ -95,17 +96,19 @@ class DetectionMAPOpKernel : public framework::OpKernel<T> {
 
     if (in_pos_count != nullptr && state) {
       GetInputPos(*in_pos_count, *in_true_pos, *in_false_pos, label_pos_count,
-                  true_pos, false_pos);
+                  true_pos, false_pos, class_num);
     }
 
     CalcTrueAndFalsePositive(gt_boxes, detect_boxes, evaluate_difficult,
                              overlap_threshold, label_pos_count, true_pos,
                              false_pos);
 
-    T map = CalcMAP(ap_type, label_pos_count, true_pos, false_pos);
+    int background_label = ctx.Attr<int>("background_label");
+    T map = CalcMAP(ap_type, label_pos_count, true_pos, false_pos,
+                    background_label);
 
     GetOutputPos(ctx, label_pos_count, true_pos, false_pos, *out_pos_count,
-                 *out_true_pos, *out_false_pos);
+                 *out_true_pos, *out_false_pos, class_num);
 
     T* map_data = out_map->mutable_data<T>(ctx.GetPlace());
     map_data[0] = map;
@@ -190,24 +193,20 @@ class DetectionMAPOpKernel : public framework::OpKernel<T> {
       const std::map<int, std::vector<std::pair<T, int>>>& false_pos,
       framework::Tensor& output_pos_count,
       framework::LoDTensor& output_true_pos,
-      framework::LoDTensor& output_false_pos) const {
-    int max_class_id = 0;
+      framework::LoDTensor& output_false_pos, const int class_num) const {
     int true_pos_count = 0;
     int false_pos_count = 0;
-    for (auto it = label_pos_count.begin(); it != label_pos_count.end(); ++it) {
-      int label = it->first;
-      if (label > max_class_id) max_class_id = label;
-      int label_num_pos = it->second;
-      if (label_num_pos == 0 || true_pos.find(label) == true_pos.end())
-        continue;
-      auto label_true_pos = true_pos.find(label)->second;
-      auto label_false_pos = false_pos.find(label)->second;
-      true_pos_count += label_true_pos.size();
-      false_pos_count += label_false_pos.size();
+    for (auto it = true_pos.begin(); it != true_pos.end(); ++it) {
+      auto tp = it->second;
+      true_pos_count += tp.size();
+    }
+    for (auto it = false_pos.begin(); it != false_pos.end(); ++it) {
+      auto fp = it->second;
+      false_pos_count += fp.size();
     }
 
     int* pos_count_data = output_pos_count.mutable_data<int>(
-        framework::make_ddim({max_class_id + 1, 1}), ctx.GetPlace());
+        framework::make_ddim({class_num, 1}), ctx.GetPlace());
 
     T* true_pos_data = output_true_pos.mutable_data<T>(
         framework::make_ddim({true_pos_count, 2}), ctx.GetPlace());
@@ -217,7 +216,7 @@ class DetectionMAPOpKernel : public framework::OpKernel<T> {
     false_pos_count = 0;
     std::vector<size_t> true_pos_starts = {0};
     std::vector<size_t> false_pos_starts = {0};
-    for (int i = 0; i <= max_class_id; ++i) {
+    for (int i = 0; i < class_num; ++i) {
       auto it_count = label_pos_count.find(i);
       pos_count_data[i] = 0;
       if (it_count != label_pos_count.end()) {
@@ -258,17 +257,16 @@ class DetectionMAPOpKernel : public framework::OpKernel<T> {
     return;
   }
 
-  void GetInputPos(
-      const framework::Tensor& input_pos_count,
-      const framework::LoDTensor& input_true_pos,
-      const framework::LoDTensor& input_false_pos,
-      std::map<int, int>& label_pos_count,
-      std::map<int, std::vector<std::pair<T, int>>>& true_pos,
-      std::map<int, std::vector<std::pair<T, int>>>& false_pos) const {
+  void GetInputPos(const framework::Tensor& input_pos_count,
+                   const framework::LoDTensor& input_true_pos,
+                   const framework::LoDTensor& input_false_pos,
+                   std::map<int, int>& label_pos_count,
+                   std::map<int, std::vector<std::pair<T, int>>>& true_pos,
+                   std::map<int, std::vector<std::pair<T, int>>>& false_pos,
+                   const int class_num) const {
     constexpr T kEPS = static_cast<T>(1e-6);
-    int class_number = input_pos_count.dims()[0];
     const int* pos_count_data = input_pos_count.data<int>();
-    for (int i = 0; i < class_number; ++i) {
+    for (int i = 0; i < class_num; ++i) {
       label_pos_count[i] = pos_count_data[i];
     }
 
@@ -391,17 +389,19 @@ class DetectionMAPOpKernel : public framework::OpKernel<T> {
     }
   }
 
-  T CalcMAP(
-      APType ap_type, const std::map<int, int>& label_pos_count,
-      const std::map<int, std::vector<std::pair<T, int>>>& true_pos,
-      const std::map<int, std::vector<std::pair<T, int>>>& false_pos) const {
+  T CalcMAP(APType ap_type, const std::map<int, int>& label_pos_count,
+            const std::map<int, std::vector<std::pair<T, int>>>& true_pos,
+            const std::map<int, std::vector<std::pair<T, int>>>& false_pos,
+            const int background_label) const {
     T mAP = 0.0;
     int count = 0;
     for (auto it = label_pos_count.begin(); it != label_pos_count.end(); ++it) {
       int label = it->first;
       int label_num_pos = it->second;
-      if (label_num_pos == 0 || true_pos.find(label) == true_pos.end())
+      if (label_num_pos == background_label ||
+          true_pos.find(label) == true_pos.end()) {
         continue;
+      }
       auto label_true_pos = true_pos.find(label)->second;
       auto label_false_pos = false_pos.find(label)->second;
       // Compute average precision.
@@ -450,7 +450,7 @@ class DetectionMAPOpKernel : public framework::OpKernel<T> {
       }
     }
     if (count != 0) mAP /= count;
-    return mAP * 100;
+    return mAP;
   }
 };  // namespace operators
 

paddle/fluid/operators/multiclass_nms_op.cc

@@ -324,7 +324,7 @@ class MultiClassNMSOpMaker : public framework::OpProtoAndCheckerMaker {
              " Please note, M is equal to the 1st dimension of BBoxes. ");
     AddAttr<int>(
         "background_label",
-        "(int64_t, defalut: 0) "
+        "(int, defalut: 0) "
         "The index of background label, the background label will be ignored. "
         "If set to -1, then all categories will be considered.")
         .SetDefault(0);

python/paddle/fluid/evaluator.py

@@ -312,6 +312,10 @@ class DetectionMAP(Evaluator):
             bounding box (bbox), which is a LoDTensor [N, 1].
         gt_box (Variable): The ground truth bounding box (bbox), which is a
             LoDTensor with shape [N, 6]. The layout is [xmin, ymin, xmax, ymax].
+        class_num (int): The class number.
+        background_label (int): The index of background label, the background
+            label will be ignored. If set to -1, then all categories will be
+            considered, 0 by defalut.
         overlap_threshold (float): The threshold for deciding true/false
             positive, 0.5 by defalut.
         evaluate_difficult (bool): Whether to consider difficult ground truth
@@ -345,6 +349,8 @@ def __init__(self,
                  gt_label,
                  gt_box,
                  gt_difficult,
+                 class_num,
+                 background_label=0,
                  overlap_threshold=0.5,
                  evaluate_difficult=True,
                  ap_version='integral'):
@@ -358,6 +364,8 @@ def __init__(self,
         map = layers.detection_map(
             input,
             label,
+            class_num,
+            background_label,
             overlap_threshold=overlap_threshold,
             evaluate_difficult=evaluate_difficult,
             ap_version=ap_version)
@@ -377,6 +385,8 @@ def __init__(self,
         accum_map = layers.detection_map(
             input,
             label,
+            class_num,
+            background_label,
             overlap_threshold=overlap_threshold,
             evaluate_difficult=evaluate_difficult,
             has_state=self.has_state,

python/paddle/fluid/layers/detection.py

@@ -151,6 +151,8 @@ class number, M is number of bounding boxes. For each category
 @autodoc()
 def detection_map(detect_res,
                   label,
+                  class_num,
+                  background_label=0,
                   overlap_threshold=0.3,
                   evaluate_difficult=True,
                   has_state=None,
@@ -192,7 +194,8 @@ def __create_var(type):
         attrs={
             'overlap_threshold': overlap_threshold,
             'evaluate_difficult': evaluate_difficult,
-            'ap_type': ap_version
+            'ap_type': ap_version,
+            'class_num': class_num,
         })
     return map_out
 

python/paddle/fluid/tests/test_detection.py

@@ -158,7 +158,7 @@ def test_detection_map(self):
                 append_batch_size=False,
                 dtype='float32')
 
-            map_out = layers.detection_map(detect_res=detect_res, label=label)
+            map_out = layers.detection_map(detect_res, label, 21)
             self.assertIsNotNone(map_out)
             self.assertEqual(map_out.shape, (1, ))
         print(str(program))

python/paddle/fluid/tests/unittests/test_detection_map_op.py

@@ -22,8 +22,8 @@
 
 class TestDetectionMAPOp(OpTest):
     def set_data(self):
+        self.class_num = 4
         self.init_test_case()
-
         self.mAP = [self.calc_map(self.tf_pos, self.tf_pos_lod)]
         self.label = np.array(self.label).astype('float32')
         self.detect = np.array(self.detect).astype('float32')
@@ -53,7 +53,8 @@ def set_data(self):
         self.attrs = {
             'overlap_threshold': self.overlap_threshold,
             'evaluate_difficult': self.evaluate_difficult,
-            'ap_type': self.ap_type
+            'ap_type': self.ap_type,
+            'class_num': self.class_num
         }
 
         self.out_class_pos_count = np.array(self.out_class_pos_count).astype(
@@ -126,12 +127,7 @@ def get_input_pos(class_pos_count, true_pos, true_pos_lod, false_pos,
             return class_pos_count_dict, true_pos_dict, false_pos_dict
 
         def get_output_pos(label_count, true_pos, false_pos):
-            max_label = 0
-            for (label, label_pos_num) in label_count.items():
-                if max_label < label:
-                    max_label = label
-
-            label_number = max_label + 1
+            label_number = self.class_num
 
             out_class_pos_count = []
             out_true_pos_lod = [0]
@@ -220,11 +216,16 @@ def get_accumulation(pos_list):
 
                 mAP += average_precisions
                 count += 1
-        self.out_class_pos_count, self.out_true_pos, self.out_true_pos_lod, self.out_false_pos, self.out_false_pos_lod = get_output_pos(
-            label_count, true_pos, false_pos)
+        pcnt, tp, tp_lod, fp, fp_lod = get_output_pos(label_count, true_pos,
+                                                      false_pos)
+        self.out_class_pos_count = pcnt
+        self.out_true_pos = tp
+        self.out_true_pos_lod = tp_lod
+        self.out_false_pos = fp
+        self.out_false_pos_lod = fp_lod
         if count != 0:
             mAP /= count
-        return mAP * 100.0
+        return mAP
 
     def setUp(self):
         self.op_type = "detection_map"