Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
1 contributor

Users who have contributed to this file

427 lines (410 sloc) 19.4 KB
###########################################################################################
# #
# Evaluator class: Implements the most popular metrics for object detection #
# #
# Developed by: Rafael Padilla (rafael.padilla@smt.ufrj.br) #
# SMT - Signal Multimedia and Telecommunications Lab #
# COPPE - Universidade Federal do Rio de Janeiro #
# Last modification: Oct 9th 2018 #
###########################################################################################
import os
import sys
from collections import Counter
import matplotlib.pyplot as plt
import numpy as np
from BoundingBox import *
from BoundingBoxes import *
from utils import *
class Evaluator:
def GetPascalVOCMetrics(self,
boundingboxes,
IOUThreshold=0.5,
method=MethodAveragePrecision.EveryPointInterpolation):
"""Get the metrics used by the VOC Pascal 2012 challenge.
Get
Args:
boundingboxes: Object of the class BoundingBoxes representing ground truth and detected
bounding boxes;
IOUThreshold: IOU threshold indicating which detections will be considered TP or FP
(default value = 0.5);
method (default = EveryPointInterpolation): It can be calculated as the implementation
in the official PASCAL VOC toolkit (EveryPointInterpolation), or applying the 11-point
interpolatio as described in the paper "The PASCAL Visual Object Classes(VOC) Challenge"
or EveryPointInterpolation" (ElevenPointInterpolation);
Returns:
A list of dictionaries. Each dictionary contains information and metrics of each class.
The keys of each dictionary are:
dict['class']: class representing the current dictionary;
dict['precision']: array with the precision values;
dict['recall']: array with the recall values;
dict['AP']: average precision;
dict['interpolated precision']: interpolated precision values;
dict['interpolated recall']: interpolated recall values;
dict['total positives']: total number of ground truth positives;
dict['total TP']: total number of True Positive detections;
dict['total FP']: total number of False Negative detections;
"""
ret = [] # list containing metrics (precision, recall, average precision) of each class
# List with all ground truths (Ex: [imageName,class,confidence=1, (bb coordinates XYX2Y2)])
groundTruths = []
# List with all detections (Ex: [imageName,class,confidence,(bb coordinates XYX2Y2)])
detections = []
# Get all classes
classes = []
# Loop through all bounding boxes and separate them into GTs and detections
for bb in boundingboxes.getBoundingBoxes():
# [imageName, class, confidence, (bb coordinates XYX2Y2)]
if bb.getBBType() == BBType.GroundTruth:
groundTruths.append([
bb.getImageName(),
bb.getClassId(), 1,
bb.getAbsoluteBoundingBox(BBFormat.XYX2Y2)
])
else:
detections.append([
bb.getImageName(),
bb.getClassId(),
bb.getConfidence(),
bb.getAbsoluteBoundingBox(BBFormat.XYX2Y2)
])
# get class
if bb.getClassId() not in classes:
classes.append(bb.getClassId())
classes = sorted(classes)
# Precision x Recall is obtained individually by each class
# Loop through by classes
for c in classes:
# Get only detection of class c
dects = []
[dects.append(d) for d in detections if d[1] == c]
# Get only ground truths of class c
gts = []
[gts.append(g) for g in groundTruths if g[1] == c]
npos = len(gts)
# sort detections by decreasing confidence
dects = sorted(dects, key=lambda conf: conf[2], reverse=True)
TP = np.zeros(len(dects))
FP = np.zeros(len(dects))
# create dictionary with amount of gts for each image
det = Counter([cc[0] for cc in gts])
for key, val in det.items():
det[key] = np.zeros(val)
# print("Evaluating class: %s (%d detections)" % (str(c), len(dects)))
# Loop through detections
for d in range(len(dects)):
# print('dect %s => %s' % (dects[d][0], dects[d][3],))
# Find ground truth image
gt = [gt for gt in gts if gt[0] == dects[d][0]]
iouMax = sys.float_info.min
for j in range(len(gt)):
# print('Ground truth gt => %s' % (gt[j][3],))
iou = Evaluator.iou(dects[d][3], gt[j][3])
if iou > iouMax:
iouMax = iou
jmax = j
# Assign detection as true positive/don't care/false positive
if iouMax >= IOUThreshold:
if det[dects[d][0]][jmax] == 0:
TP[d] = 1 # count as true positive
det[dects[d][0]][jmax] = 1 # flag as already 'seen'
# print("TP")
else:
FP[d] = 1 # count as false positive
# print("FP")
# - A detected "cat" is overlaped with a GT "cat" with IOU >= IOUThreshold.
else:
FP[d] = 1 # count as false positive
# print("FP")
# compute precision, recall and average precision
acc_FP = np.cumsum(FP)
acc_TP = np.cumsum(TP)
rec = acc_TP / npos
prec = np.divide(acc_TP, (acc_FP + acc_TP))
# Depending on the method, call the right implementation
if method == MethodAveragePrecision.EveryPointInterpolation:
[ap, mpre, mrec, ii] = Evaluator.CalculateAveragePrecision(rec, prec)
else:
[ap, mpre, mrec, _] = Evaluator.ElevenPointInterpolatedAP(rec, prec)
# add class result in the dictionary to be returned
r = {
'class': c,
'precision': prec,
'recall': rec,
'AP': ap,
'interpolated precision': mpre,
'interpolated recall': mrec,
'total positives': npos,
'total TP': np.sum(TP),
'total FP': np.sum(FP)
}
ret.append(r)
return ret
def PlotPrecisionRecallCurve(self,
boundingBoxes,
IOUThreshold=0.5,
method=MethodAveragePrecision.EveryPointInterpolation,
showAP=False,
showInterpolatedPrecision=False,
savePath=None,
showGraphic=True):
"""PlotPrecisionRecallCurve
Plot the Precision x Recall curve for a given class.
Args:
boundingBoxes: Object of the class BoundingBoxes representing ground truth and detected
bounding boxes;
IOUThreshold (optional): IOU threshold indicating which detections will be considered
TP or FP (default value = 0.5);
method (default = EveryPointInterpolation): It can be calculated as the implementation
in the official PASCAL VOC toolkit (EveryPointInterpolation), or applying the 11-point
interpolatio as described in the paper "The PASCAL Visual Object Classes(VOC) Challenge"
or EveryPointInterpolation" (ElevenPointInterpolation).
showAP (optional): if True, the average precision value will be shown in the title of
the graph (default = False);
showInterpolatedPrecision (optional): if True, it will show in the plot the interpolated
precision (default = False);
savePath (optional): if informed, the plot will be saved as an image in this path
(ex: /home/mywork/ap.png) (default = None);
showGraphic (optional): if True, the plot will be shown (default = True)
Returns:
A list of dictionaries. Each dictionary contains information and metrics of each class.
The keys of each dictionary are:
dict['class']: class representing the current dictionary;
dict['precision']: array with the precision values;
dict['recall']: array with the recall values;
dict['AP']: average precision;
dict['interpolated precision']: interpolated precision values;
dict['interpolated recall']: interpolated recall values;
dict['total positives']: total number of ground truth positives;
dict['total TP']: total number of True Positive detections;
dict['total FP']: total number of False Negative detections;
"""
results = self.GetPascalVOCMetrics(boundingBoxes, IOUThreshold, method)
result = None
# Each resut represents a class
for result in results:
if result is None:
raise IOError('Error: Class %d could not be found.' % classId)
classId = result['class']
precision = result['precision']
recall = result['recall']
average_precision = result['AP']
mpre = result['interpolated precision']
mrec = result['interpolated recall']
npos = result['total positives']
total_tp = result['total TP']
total_fp = result['total FP']
plt.close()
if showInterpolatedPrecision:
if method == MethodAveragePrecision.EveryPointInterpolation:
plt.plot(mrec, mpre, '--r', label='Interpolated precision (every point)')
elif method == MethodAveragePrecision.ElevenPointInterpolation:
# Uncomment the line below if you want to plot the area
# plt.plot(mrec, mpre, 'or', label='11-point interpolated precision')
# Remove duplicates, getting only the highest precision of each recall value
nrec = []
nprec = []
for idx in range(len(mrec)):
r = mrec[idx]
if r not in nrec:
idxEq = np.argwhere(mrec == r)
nrec.append(r)
nprec.append(max([mpre[int(id)] for id in idxEq]))
plt.plot(nrec, nprec, 'or', label='11-point interpolated precision')
plt.plot(recall, precision, label='Precision')
plt.xlabel('recall')
plt.ylabel('precision')
if showAP:
ap_str = "{0:.2f}%".format(average_precision * 100)
# ap_str = "{0:.4f}%".format(average_precision * 100)
plt.title('Precision x Recall curve \nClass: %s, AP: %s' % (str(classId), ap_str))
else:
plt.title('Precision x Recall curve \nClass: %s' % str(classId))
plt.legend(shadow=True)
plt.grid()
############################################################
# Uncomment the following block to create plot with points #
############################################################
# plt.plot(recall, precision, 'bo')
# labels = ['R', 'Y', 'J', 'A', 'U', 'C', 'M', 'F', 'D', 'B', 'H', 'P', 'E', 'X', 'N', 'T',
# 'K', 'Q', 'V', 'I', 'L', 'S', 'G', 'O']
# dicPosition = {}
# dicPosition['left_zero'] = (-30,0)
# dicPosition['left_zero_slight'] = (-30,-10)
# dicPosition['right_zero'] = (30,0)
# dicPosition['left_up'] = (-30,20)
# dicPosition['left_down'] = (-30,-25)
# dicPosition['right_up'] = (20,20)
# dicPosition['right_down'] = (20,-20)
# dicPosition['up_zero'] = (0,30)
# dicPosition['up_right'] = (0,30)
# dicPosition['left_zero_long'] = (-60,-2)
# dicPosition['down_zero'] = (-2,-30)
# vecPositions = [
# dicPosition['left_down'],
# dicPosition['left_zero'],
# dicPosition['right_zero'],
# dicPosition['right_zero'], #'R', 'Y', 'J', 'A',
# dicPosition['left_up'],
# dicPosition['left_up'],
# dicPosition['right_up'],
# dicPosition['left_up'], # 'U', 'C', 'M', 'F',
# dicPosition['left_zero'],
# dicPosition['right_up'],
# dicPosition['right_down'],
# dicPosition['down_zero'], #'D', 'B', 'H', 'P'
# dicPosition['left_up'],
# dicPosition['up_zero'],
# dicPosition['right_up'],
# dicPosition['left_up'], # 'E', 'X', 'N', 'T',
# dicPosition['left_zero'],
# dicPosition['right_zero'],
# dicPosition['left_zero_long'],
# dicPosition['left_zero_slight'], # 'K', 'Q', 'V', 'I',
# dicPosition['right_down'],
# dicPosition['left_down'],
# dicPosition['right_up'],
# dicPosition['down_zero']
# ] # 'L', 'S', 'G', 'O'
# for idx in range(len(labels)):
# box = dict(boxstyle='round,pad=.5',facecolor='yellow',alpha=0.5)
# plt.annotate(labels[idx],
# xy=(recall[idx],precision[idx]), xycoords='data',
# xytext=vecPositions[idx], textcoords='offset points',
# arrowprops=dict(arrowstyle="->", connectionstyle="arc3"),
# bbox=box)
if savePath is not None:
plt.savefig(os.path.join(savePath, classId + '.png'))
if showGraphic is True:
plt.show()
# plt.waitforbuttonpress()
plt.pause(0.05)
return results
@staticmethod
def CalculateAveragePrecision(rec, prec):
mrec = []
mrec.append(0)
[mrec.append(e) for e in rec]
mrec.append(1)
mpre = []
mpre.append(0)
[mpre.append(e) for e in prec]
mpre.append(0)
for i in range(len(mpre) - 1, 0, -1):
mpre[i - 1] = max(mpre[i - 1], mpre[i])
ii = []
for i in range(len(mrec) - 1):
if mrec[1:][i] != mrec[0:-1][i]:
ii.append(i + 1)
ap = 0
for i in ii:
ap = ap + np.sum((mrec[i] - mrec[i - 1]) * mpre[i])
# return [ap, mpre[1:len(mpre)-1], mrec[1:len(mpre)-1], ii]
return [ap, mpre[0:len(mpre) - 1], mrec[0:len(mpre) - 1], ii]
@staticmethod
# 11-point interpolated average precision
def ElevenPointInterpolatedAP(rec, prec):
# def CalculateAveragePrecision2(rec, prec):
mrec = []
# mrec.append(0)
[mrec.append(e) for e in rec]
# mrec.append(1)
mpre = []
# mpre.append(0)
[mpre.append(e) for e in prec]
# mpre.append(0)
recallValues = np.linspace(0, 1, 11)
recallValues = list(recallValues[::-1])
rhoInterp = []
recallValid = []
# For each recallValues (0, 0.1, 0.2, ... , 1)
for r in recallValues:
# Obtain all recall values higher or equal than r
argGreaterRecalls = np.argwhere(mrec[:] >= r)
pmax = 0
# If there are recalls above r
if argGreaterRecalls.size != 0:
pmax = max(mpre[argGreaterRecalls.min():])
recallValid.append(r)
rhoInterp.append(pmax)
# By definition AP = sum(max(precision whose recall is above r))/11
ap = sum(rhoInterp) / 11
# Generating values for the plot
rvals = []
rvals.append(recallValid[0])
[rvals.append(e) for e in recallValid]
rvals.append(0)
pvals = []
pvals.append(0)
[pvals.append(e) for e in rhoInterp]
pvals.append(0)
# rhoInterp = rhoInterp[::-1]
cc = []
for i in range(len(rvals)):
p = (rvals[i], pvals[i - 1])
if p not in cc:
cc.append(p)
p = (rvals[i], pvals[i])
if p not in cc:
cc.append(p)
recallValues = [i[0] for i in cc]
rhoInterp = [i[1] for i in cc]
return [ap, rhoInterp, recallValues, None]
# For each detections, calculate IOU with reference
@staticmethod
def _getAllIOUs(reference, detections):
ret = []
bbReference = reference.getAbsoluteBoundingBox(BBFormat.XYX2Y2)
# img = np.zeros((200,200,3), np.uint8)
for d in detections:
bb = d.getAbsoluteBoundingBox(BBFormat.XYX2Y2)
iou = Evaluator.iou(bbReference, bb)
# Show blank image with the bounding boxes
# img = add_bb_into_image(img, d, color=(255,0,0), thickness=2, label=None)
# img = add_bb_into_image(img, reference, color=(0,255,0), thickness=2, label=None)
ret.append((iou, reference, d)) # iou, reference, detection
# cv2.imshow("comparing",img)
# cv2.waitKey(0)
# cv2.destroyWindow("comparing")
return sorted(ret, key=lambda i: i[0], reverse=True) # sort by iou (from highest to lowest)
@staticmethod
def iou(boxA, boxB):
# if boxes dont intersect
if Evaluator._boxesIntersect(boxA, boxB) is False:
return 0
interArea = Evaluator._getIntersectionArea(boxA, boxB)
union = Evaluator._getUnionAreas(boxA, boxB, interArea=interArea)
# intersection over union
iou = interArea / union
assert iou >= 0
return iou
# boxA = (Ax1,Ay1,Ax2,Ay2)
# boxB = (Bx1,By1,Bx2,By2)
@staticmethod
def _boxesIntersect(boxA, boxB):
if boxA[0] > boxB[2]:
return False # boxA is right of boxB
if boxB[0] > boxA[2]:
return False # boxA is left of boxB
if boxA[3] < boxB[1]:
return False # boxA is above boxB
if boxA[1] > boxB[3]:
return False # boxA is below boxB
return True
@staticmethod
def _getIntersectionArea(boxA, boxB):
xA = max(boxA[0], boxB[0])
yA = max(boxA[1], boxB[1])
xB = min(boxA[2], boxB[2])
yB = min(boxA[3], boxB[3])
# intersection area
return (xB - xA + 1) * (yB - yA + 1)
@staticmethod
def _getUnionAreas(boxA, boxB, interArea=None):
area_A = Evaluator._getArea(boxA)
area_B = Evaluator._getArea(boxB)
if interArea is None:
interArea = Evaluator._getIntersectionArea(boxA, boxB)
return float(area_A + area_B - interArea)
@staticmethod
def _getArea(box):
return (box[2] - box[0] + 1) * (box[3] - box[1] + 1)
You can’t perform that action at this time.