forked from johnnynunez/detectron2
-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathtest_coco_evaluation.py
138 lines (126 loc) · 8.59 KB
/
test_coco_evaluation.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
# Copyright (c) Facebook, Inc. and its affiliates.
import contextlib
import copy
import io
import json
import numpy as np
import os
import tempfile
import unittest
import torch
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
from detectron2.data import DatasetCatalog
from detectron2.evaluation import COCOEvaluator
from detectron2.evaluation.fast_eval_api import COCOeval_opt
from detectron2.structures import Boxes, Instances
class TestCOCOeval(unittest.TestCase):
def test_fast_eval(self):
# A small set of images/categories from COCO val
# fmt: off
detections = [{"image_id": 139, "category_id": 1, "bbox": [417.3332824707031, 159.27003479003906, 47.66064453125, 143.00193786621094], "score": 0.9949821829795837, "segmentation": {"size": [426, 640], "counts": "Tc`52W=3N0N4aNN^E7]:4XE1g:8kDMT;U100000001O1gE[Nk8h1dFiNY9Z1aFkN]9g2J3NdN`FlN`9S1cFRN07]9g1bFoM6;X9c1cFoM=8R9g1bFQN>3U9Y30O01OO1O001N2O1N1O4L4L5UNoE3V:CVF6Q:@YF9l9@ZF<k9[O`F=];HYnX2"}}, {"image_id": 139, "category_id": 1, "bbox": [383.5909118652344, 172.0777587890625, 17.959075927734375, 36.94813537597656], "score": 0.7685421705245972, "segmentation": {"size": [426, 640], "counts": "lZP5m0Z<300O100O100000001O00]OlC0T<OnCOT<OnCNX<JnC2bQT3"}}, {"image_id": 139, "category_id": 1, "bbox": [457.8359069824219, 158.88027954101562, 9.89764404296875, 8.771820068359375], "score": 0.07092753797769547, "segmentation": {"size": [426, 640], "counts": "bSo54T=2N2O1001O006ImiW2"}}] # noqa
gt_annotations = {"categories": [{"supercategory": "person", "id": 1, "name": "person"}, {"supercategory": "furniture", "id": 65, "name": "bed"}], "images": [{"license": 4, "file_name": "000000000285.jpg", "coco_url": "http://images.cocodataset.org/val2017/000000000285.jpg", "height": 640, "width": 586, "date_captured": "2013-11-18 13:09:47", "flickr_url": "http://farm8.staticflickr.com/7434/9138147604_c6225224b8_z.jpg", "id": 285}, {"license": 2, "file_name": "000000000139.jpg", "coco_url": "http://images.cocodataset.org/val2017/000000000139.jpg", "height": 426, "width": 640, "date_captured": "2013-11-21 01:34:01", "flickr_url": "http://farm9.staticflickr.com/8035/8024364858_9c41dc1666_z.jpg", "id": 139}], "annotations": [{"segmentation": [[428.19, 219.47, 430.94, 209.57, 430.39, 210.12, 421.32, 216.17, 412.8, 217.27, 413.9, 214.24, 422.42, 211.22, 429.29, 201.6, 430.67, 181.8, 430.12, 175.2, 427.09, 168.06, 426.27, 164.21, 430.94, 159.26, 440.29, 157.61, 446.06, 163.93, 448.53, 168.06, 448.53, 173.01, 449.08, 174.93, 454.03, 185.1, 455.41, 188.4, 458.43, 195.0, 460.08, 210.94, 462.28, 226.61, 460.91, 233.76, 454.31, 234.04, 460.08, 256.85, 462.56, 268.13, 465.58, 290.67, 465.85, 293.14, 463.38, 295.62, 452.66, 295.34, 448.26, 294.52, 443.59, 282.7, 446.06, 235.14, 446.34, 230.19, 438.09, 232.39, 438.09, 221.67, 434.24, 221.12, 427.09, 219.74]], "area": 2913.1103999999987, "iscrowd": 0, "image_id": 139, "bbox": [412.8, 157.61, 53.05, 138.01], "category_id": 1, "id": 230831}, {"segmentation": [[384.98, 206.58, 384.43, 199.98, 385.25, 193.66, 385.25, 190.08, 387.18, 185.13, 387.18, 182.93, 386.08, 181.01, 385.25, 178.81, 385.25, 175.79, 388.0, 172.76, 394.88, 172.21, 398.72, 173.31, 399.27, 176.06, 399.55, 183.48, 397.9, 185.68, 395.15, 188.98, 396.8, 193.38, 398.45, 194.48, 399.0, 205.75, 395.43, 207.95, 388.83, 206.03]], "area": 435.1449499999997, "iscrowd": 0, "image_id": 139, "bbox": [384.43, 172.21, 15.12, 35.74], "category_id": 1, "id": 233201}]} # noqa
# fmt: on
# Test a small dataset for typical COCO format
experiments = {"full": (detections, gt_annotations, {})}
# Test what happens if the list of detections or ground truth annotations is empty
experiments["empty_dt"] = ([], gt_annotations, {})
gt = copy.deepcopy(gt_annotations)
gt["annotations"] = []
experiments["empty_gt"] = (detections, gt, {})
# Test changing parameter settings
experiments["no_categories"] = (detections, gt_annotations, {"useCats": 0})
experiments["no_ious"] = (detections, gt_annotations, {"iouThrs": []})
experiments["no_rec_thrs"] = (detections, gt_annotations, {"recThrs": []})
experiments["no_max_dets"] = (detections, gt_annotations, {"maxDets": []})
experiments["one_max_det"] = (detections, gt_annotations, {"maxDets": [1]})
experiments["no_area"] = (detections, gt_annotations, {"areaRng": [], "areaRngLbl": []})
# Test what happens if one omits different fields from the annotation structure
annotation_fields = [
"id",
"image_id",
"category_id",
"score",
"area",
"iscrowd",
"ignore",
"bbox",
"segmentation",
]
for a in annotation_fields:
gt = copy.deepcopy(gt_annotations)
for g in gt["annotations"]:
if a in g:
del g[a]
dt = copy.deepcopy(detections)
for d in dt:
if a in d:
del d[a]
experiments["omit_gt_" + a] = (detections, gt, {})
experiments["omit_dt_" + a] = (dt, gt_annotations, {})
# Compare precision/recall for original COCO PythonAPI to custom optimized one
for name, (dt, gt, params) in experiments.items():
# Dump to json.
try:
with tempfile.TemporaryDirectory() as tmpdir:
json_file_name = os.path.join(tmpdir, "gt_" + name + ".json")
with open(json_file_name, "w") as f:
json.dump(gt, f)
with contextlib.redirect_stdout(io.StringIO()):
coco_api = COCO(json_file_name)
except Exception:
pass
for iou_type in ["bbox", "segm", "keypoints"]:
# Run original COCOeval PythonAPI
api_exception = None
try:
with contextlib.redirect_stdout(io.StringIO()):
coco_dt = coco_api.loadRes(dt)
coco_eval = COCOeval(coco_api, coco_dt, iou_type)
for p, v in params.items():
setattr(coco_eval.params, p, v)
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
except Exception as ex:
api_exception = ex
# Run optimized COCOeval_opt API
opt_exception = None
try:
with contextlib.redirect_stdout(io.StringIO()):
coco_dt = coco_api.loadRes(dt)
coco_eval_opt = COCOeval_opt(coco_api, coco_dt, iou_type)
for p, v in params.items():
setattr(coco_eval_opt.params, p, v)
coco_eval_opt.evaluate()
coco_eval_opt.accumulate()
coco_eval_opt.summarize()
except Exception as ex:
opt_exception = ex
if api_exception is not None and opt_exception is not None:
# Original API and optimized API should throw the same exception if annotation
# format is bad
api_error = "" if api_exception is None else type(api_exception).__name__
opt_error = "" if opt_exception is None else type(opt_exception).__name__
msg = "%s: comparing COCO APIs, '%s' != '%s'" % (name, api_error, opt_error)
self.assertTrue(api_error == opt_error, msg=msg)
else:
# Original API and optimized API should produce the same precision/recalls
for k in ["precision", "recall"]:
diff = np.abs(coco_eval.eval[k] - coco_eval_opt.eval[k])
abs_diff = np.max(diff) if diff.size > 0 else 0.0
msg = "%s: comparing COCO APIs, %s differs by %f" % (name, k, abs_diff)
self.assertTrue(abs_diff < 1e-4, msg=msg)
def test_unknown_category(self):
dataset = "coco_2017_val_100"
evaluator = COCOEvaluator(dataset)
evaluator.reset()
inputs = DatasetCatalog.get(dataset)[:2]
pred = Instances((100, 100))
pred.pred_boxes = Boxes(torch.rand(2, 4))
pred.scores = torch.rand(2)
pred.pred_classes = torch.tensor([10, 80])
output = {"instances": pred}
evaluator.process(inputs, [output, output])
with self.assertRaises(AssertionError):
evaluator.evaluate()