From 0e1f82fd2a22f85cfc483bcdd55815e9006a3a8a Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Tue, 6 Mar 2018 18:34:02 +0800 Subject: [PATCH] Fix bug in detection mAP evaluator. (#8778) * Fix mAP evaluator bug. * Fix bug in detection mAP evaluator. * Fix unit testing. * Support to set background label index in detection mAP op. --- paddle/fluid/operators/detection_map_op.cc | 10 ++- paddle/fluid/operators/detection_map_op.h | 64 +++++++++---------- paddle/fluid/operators/multiclass_nms_op.cc | 2 +- python/paddle/fluid/evaluator.py | 10 +++ python/paddle/fluid/layers/detection.py | 5 +- python/paddle/fluid/tests/test_detection.py | 2 +- .../tests/unittests/test_detection_map_op.py | 23 +++---- 7 files changed, 69 insertions(+), 47 deletions(-) diff --git a/paddle/fluid/operators/detection_map_op.cc b/paddle/fluid/operators/detection_map_op.cc index 880bfe3b04394..9b8ca925373eb 100644 --- a/paddle/fluid/operators/detection_map_op.cc +++ b/paddle/fluid/operators/detection_map_op.cc @@ -142,7 +142,15 @@ class DetectionMAPOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("MAP", "(Tensor) A tensor with shape [1], store the mAP evaluate " "result of the detection."); - + AddAttr("class_num", + "(int) " + "The class number."); + AddAttr( + "background_label", + "(int, defalut: 0) " + "The index of background label, the background label will be ignored. " + "If set to -1, then all categories will be considered.") + .SetDefault(0); AddAttr( "overlap_threshold", "(float) " diff --git a/paddle/fluid/operators/detection_map_op.h b/paddle/fluid/operators/detection_map_op.h index b2b0995b35bf1..637f8368f8889 100644 --- a/paddle/fluid/operators/detection_map_op.h +++ b/paddle/fluid/operators/detection_map_op.h @@ -69,6 +69,7 @@ class DetectionMAPOpKernel : public framework::OpKernel { float overlap_threshold = ctx.Attr("overlap_threshold"); float evaluate_difficult = ctx.Attr("evaluate_difficult"); auto ap_type = GetAPType(ctx.Attr("ap_type")); + int class_num = ctx.Attr("class_num"); auto label_lod = in_label->lod(); auto detect_lod = in_detect->lod(); @@ -95,17 +96,19 @@ class DetectionMAPOpKernel : public framework::OpKernel { if (in_pos_count != nullptr && state) { GetInputPos(*in_pos_count, *in_true_pos, *in_false_pos, label_pos_count, - true_pos, false_pos); + true_pos, false_pos, class_num); } CalcTrueAndFalsePositive(gt_boxes, detect_boxes, evaluate_difficult, overlap_threshold, label_pos_count, true_pos, false_pos); - T map = CalcMAP(ap_type, label_pos_count, true_pos, false_pos); + int background_label = ctx.Attr("background_label"); + T map = CalcMAP(ap_type, label_pos_count, true_pos, false_pos, + background_label); GetOutputPos(ctx, label_pos_count, true_pos, false_pos, *out_pos_count, - *out_true_pos, *out_false_pos); + *out_true_pos, *out_false_pos, class_num); T* map_data = out_map->mutable_data(ctx.GetPlace()); map_data[0] = map; @@ -190,24 +193,20 @@ class DetectionMAPOpKernel : public framework::OpKernel { const std::map>>& false_pos, framework::Tensor& output_pos_count, framework::LoDTensor& output_true_pos, - framework::LoDTensor& output_false_pos) const { - int max_class_id = 0; + framework::LoDTensor& output_false_pos, const int class_num) const { int true_pos_count = 0; int false_pos_count = 0; - for (auto it = label_pos_count.begin(); it != label_pos_count.end(); ++it) { - int label = it->first; - if (label > max_class_id) max_class_id = label; - int label_num_pos = it->second; - if (label_num_pos == 0 || true_pos.find(label) == true_pos.end()) - continue; - auto label_true_pos = true_pos.find(label)->second; - auto label_false_pos = false_pos.find(label)->second; - true_pos_count += label_true_pos.size(); - false_pos_count += label_false_pos.size(); + for (auto it = true_pos.begin(); it != true_pos.end(); ++it) { + auto tp = it->second; + true_pos_count += tp.size(); + } + for (auto it = false_pos.begin(); it != false_pos.end(); ++it) { + auto fp = it->second; + false_pos_count += fp.size(); } int* pos_count_data = output_pos_count.mutable_data( - framework::make_ddim({max_class_id + 1, 1}), ctx.GetPlace()); + framework::make_ddim({class_num, 1}), ctx.GetPlace()); T* true_pos_data = output_true_pos.mutable_data( framework::make_ddim({true_pos_count, 2}), ctx.GetPlace()); @@ -217,7 +216,7 @@ class DetectionMAPOpKernel : public framework::OpKernel { false_pos_count = 0; std::vector true_pos_starts = {0}; std::vector false_pos_starts = {0}; - for (int i = 0; i <= max_class_id; ++i) { + for (int i = 0; i < class_num; ++i) { auto it_count = label_pos_count.find(i); pos_count_data[i] = 0; if (it_count != label_pos_count.end()) { @@ -258,17 +257,16 @@ class DetectionMAPOpKernel : public framework::OpKernel { return; } - void GetInputPos( - const framework::Tensor& input_pos_count, - const framework::LoDTensor& input_true_pos, - const framework::LoDTensor& input_false_pos, - std::map& label_pos_count, - std::map>>& true_pos, - std::map>>& false_pos) const { + void GetInputPos(const framework::Tensor& input_pos_count, + const framework::LoDTensor& input_true_pos, + const framework::LoDTensor& input_false_pos, + std::map& label_pos_count, + std::map>>& true_pos, + std::map>>& false_pos, + const int class_num) const { constexpr T kEPS = static_cast(1e-6); - int class_number = input_pos_count.dims()[0]; const int* pos_count_data = input_pos_count.data(); - for (int i = 0; i < class_number; ++i) { + for (int i = 0; i < class_num; ++i) { label_pos_count[i] = pos_count_data[i]; } @@ -391,17 +389,19 @@ class DetectionMAPOpKernel : public framework::OpKernel { } } - T CalcMAP( - APType ap_type, const std::map& label_pos_count, - const std::map>>& true_pos, - const std::map>>& false_pos) const { + T CalcMAP(APType ap_type, const std::map& label_pos_count, + const std::map>>& true_pos, + const std::map>>& false_pos, + const int background_label) const { T mAP = 0.0; int count = 0; for (auto it = label_pos_count.begin(); it != label_pos_count.end(); ++it) { int label = it->first; int label_num_pos = it->second; - if (label_num_pos == 0 || true_pos.find(label) == true_pos.end()) + if (label_num_pos == background_label || + true_pos.find(label) == true_pos.end()) { continue; + } auto label_true_pos = true_pos.find(label)->second; auto label_false_pos = false_pos.find(label)->second; // Compute average precision. @@ -450,7 +450,7 @@ class DetectionMAPOpKernel : public framework::OpKernel { } } if (count != 0) mAP /= count; - return mAP * 100; + return mAP; } }; // namespace operators diff --git a/paddle/fluid/operators/multiclass_nms_op.cc b/paddle/fluid/operators/multiclass_nms_op.cc index c4e70cde6f8c6..0f80f752c95e9 100644 --- a/paddle/fluid/operators/multiclass_nms_op.cc +++ b/paddle/fluid/operators/multiclass_nms_op.cc @@ -324,7 +324,7 @@ class MultiClassNMSOpMaker : public framework::OpProtoAndCheckerMaker { " Please note, M is equal to the 1st dimension of BBoxes. "); AddAttr( "background_label", - "(int64_t, defalut: 0) " + "(int, defalut: 0) " "The index of background label, the background label will be ignored. " "If set to -1, then all categories will be considered.") .SetDefault(0); diff --git a/python/paddle/fluid/evaluator.py b/python/paddle/fluid/evaluator.py index 364789233bf9c..18b1cdce8ad88 100644 --- a/python/paddle/fluid/evaluator.py +++ b/python/paddle/fluid/evaluator.py @@ -312,6 +312,10 @@ class DetectionMAP(Evaluator): bounding box (bbox), which is a LoDTensor [N, 1]. gt_box (Variable): The ground truth bounding box (bbox), which is a LoDTensor with shape [N, 6]. The layout is [xmin, ymin, xmax, ymax]. + class_num (int): The class number. + background_label (int): The index of background label, the background + label will be ignored. If set to -1, then all categories will be + considered, 0 by defalut. overlap_threshold (float): The threshold for deciding true/false positive, 0.5 by defalut. evaluate_difficult (bool): Whether to consider difficult ground truth @@ -345,6 +349,8 @@ def __init__(self, gt_label, gt_box, gt_difficult, + class_num, + background_label=0, overlap_threshold=0.5, evaluate_difficult=True, ap_version='integral'): @@ -358,6 +364,8 @@ def __init__(self, map = layers.detection_map( input, label, + class_num, + background_label, overlap_threshold=overlap_threshold, evaluate_difficult=evaluate_difficult, ap_version=ap_version) @@ -377,6 +385,8 @@ def __init__(self, accum_map = layers.detection_map( input, label, + class_num, + background_label, overlap_threshold=overlap_threshold, evaluate_difficult=evaluate_difficult, has_state=self.has_state, diff --git a/python/paddle/fluid/layers/detection.py b/python/paddle/fluid/layers/detection.py index 420d3de7f73aa..2bf7cf21ca94c 100644 --- a/python/paddle/fluid/layers/detection.py +++ b/python/paddle/fluid/layers/detection.py @@ -151,6 +151,8 @@ class number, M is number of bounding boxes. For each category @autodoc() def detection_map(detect_res, label, + class_num, + background_label=0, overlap_threshold=0.3, evaluate_difficult=True, has_state=None, @@ -192,7 +194,8 @@ def __create_var(type): attrs={ 'overlap_threshold': overlap_threshold, 'evaluate_difficult': evaluate_difficult, - 'ap_type': ap_version + 'ap_type': ap_version, + 'class_num': class_num, }) return map_out diff --git a/python/paddle/fluid/tests/test_detection.py b/python/paddle/fluid/tests/test_detection.py index b183db55b2afa..921260ef3f4b1 100644 --- a/python/paddle/fluid/tests/test_detection.py +++ b/python/paddle/fluid/tests/test_detection.py @@ -158,7 +158,7 @@ def test_detection_map(self): append_batch_size=False, dtype='float32') - map_out = layers.detection_map(detect_res=detect_res, label=label) + map_out = layers.detection_map(detect_res, label, 21) self.assertIsNotNone(map_out) self.assertEqual(map_out.shape, (1, )) print(str(program)) diff --git a/python/paddle/fluid/tests/unittests/test_detection_map_op.py b/python/paddle/fluid/tests/unittests/test_detection_map_op.py index 9857cc58456b5..f3197a623efb1 100644 --- a/python/paddle/fluid/tests/unittests/test_detection_map_op.py +++ b/python/paddle/fluid/tests/unittests/test_detection_map_op.py @@ -22,8 +22,8 @@ class TestDetectionMAPOp(OpTest): def set_data(self): + self.class_num = 4 self.init_test_case() - self.mAP = [self.calc_map(self.tf_pos, self.tf_pos_lod)] self.label = np.array(self.label).astype('float32') self.detect = np.array(self.detect).astype('float32') @@ -53,7 +53,8 @@ def set_data(self): self.attrs = { 'overlap_threshold': self.overlap_threshold, 'evaluate_difficult': self.evaluate_difficult, - 'ap_type': self.ap_type + 'ap_type': self.ap_type, + 'class_num': self.class_num } self.out_class_pos_count = np.array(self.out_class_pos_count).astype( @@ -126,12 +127,7 @@ def get_input_pos(class_pos_count, true_pos, true_pos_lod, false_pos, return class_pos_count_dict, true_pos_dict, false_pos_dict def get_output_pos(label_count, true_pos, false_pos): - max_label = 0 - for (label, label_pos_num) in label_count.items(): - if max_label < label: - max_label = label - - label_number = max_label + 1 + label_number = self.class_num out_class_pos_count = [] out_true_pos_lod = [0] @@ -220,11 +216,16 @@ def get_accumulation(pos_list): mAP += average_precisions count += 1 - self.out_class_pos_count, self.out_true_pos, self.out_true_pos_lod, self.out_false_pos, self.out_false_pos_lod = get_output_pos( - label_count, true_pos, false_pos) + pcnt, tp, tp_lod, fp, fp_lod = get_output_pos(label_count, true_pos, + false_pos) + self.out_class_pos_count = pcnt + self.out_true_pos = tp + self.out_true_pos_lod = tp_lod + self.out_false_pos = fp + self.out_false_pos_lod = fp_lod if count != 0: mAP /= count - return mAP * 100.0 + return mAP def setUp(self): self.op_type = "detection_map"