-
Notifications
You must be signed in to change notification settings - Fork 6
/
cooccur_metric.py
153 lines (106 loc) · 5.47 KB
/
cooccur_metric.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
import numpy as np
import os
import pickle
from sklearn.metrics import precision_recall_fscore_support, average_precision_score
import warnings
def conditional_t(y_pred, y_gt, y_gt_mask, thresh=0.5, avg=True):
n_samples, n_classes = y_pred.shape
assert y_pred.shape == y_gt.shape
prec, rec = np.ones((n_classes, n_classes))*-1.0, np.ones((n_classes, n_classes))*-1.0
maps = np.ones((n_classes, n_classes))*-1.0
n_occurs = np.zeros((n_classes, n_classes))
for c_j in range(n_classes):
y_gt_sub = y_gt[y_gt_mask[:, c_j] == 1] # contains the subset of samples where c_j exists
y_pred_sub = y_pred[y_gt_mask[:, c_j] == 1]
pr_j, re_j, f1_j, n_j = precision_recall_fscore_support(y_gt_sub, (y_pred_sub >= thresh).astype(np.uint8), average=None)
map_j = average_precision_score(y_gt_sub, y_pred_sub, average=None)
for c_i in range(n_classes):
if c_i == c_j:
continue
if n_j[c_i] == 0:
continue
n_occurs[c_i, c_j] = n_j[c_i]
if np.isnan(pr_j[c_i]) or np.isnan(re_j[c_i]):
prec[c_i, c_j] = 0
rec[c_i, c_j] = 0
else:
prec[c_i, c_j] = pr_j[c_i]
rec[c_i, c_j] = re_j[c_i]
if np.isnan(map_j[c_i]):
maps[c_i, c_j] = 0
else:
maps[c_i, c_j] = map_j[c_i]
if avg:
return np.mean(prec[prec != -1]), np.mean(rec[rec != -1]), n_occurs, np.mean(maps[maps != -1])
else:
return prec, rec, n_occurs, maps
def avg_scores(score):
return np.mean(score[score >= 0])*100
def get_f1(prec, rec):
return 2*(prec*rec)/(prec+rec+1e-9)
def standard_metric(y_pred, y_gt, thresh=0.5):
y_pred = np.concatenate(y_pred, 0)
y_gt = np.concatenate(y_gt, 0)
pr, re, _, n = precision_recall_fscore_support(y_gt, (y_pred >= thresh).astype(np.uint8), average=None)
maps = average_precision_score(y_gt, y_pred, average=None)
return pr, re, n, maps
def conditional_metric(y_pred, y_gt, t=0, thresh=0.5, avg=True):
"""
y_pred is a list of un-thresholded predictions [(T1, C), (T2, C), ...]. Each element of the list is a different video, where the shape is (Time, #Classes).
y_gt is a list of binary ground-truth labels [(T1, C), (T2, C), ...]. Each element of the list is a different video, where the shape is (Time, #Classes).
t is an integer. If =0, measures in-timestep coocurrence. If >0, it measures conditional score of succeeding
actions (i.e. if c_i follows c_j). If <0 it measure conditional score of preceeding actions (i.e. if c_i preceeds c_j).
thresh is a value in range (0, 1) which binarizes the predicted probabilities
avg determines whether it returns a single score or class-wise scores
Returns
prec: the action-conditional precision score
rec: the action-conditional recall score
n_s: the number of samples for the pair of actions. Has shape (#Classes, #Classes).
map: the action-conditional mAP score
"""
y_pred = np.concatenate(y_pred, 0)
if t == 0:
y_gt = np.concatenate(y_gt, 0).astype(np.uint8)
return conditional_t(y_pred, y_gt, y_gt, thresh, avg)
else:
y_gt_mask = []
for vid_y_gt in y_gt:
if t > 0: # looks at previous t time-steps
cumsum = np.cumsum(vid_y_gt, 0)
rolled = np.roll(cumsum, t, 0)
rolled[:t] = 0
n_in_last_t = cumsum-rolled
else: # looks at next 0-t time-steps
vid_y_gt_flipped = np.flip(vid_y_gt, 0)
cumsum = np.cumsum(vid_y_gt_flipped, 0)
rolled = np.roll(cumsum, t, 0)
rolled[:0-t] = 0
n_in_last_t = cumsum-rolled
n_in_last_t = np.flip(n_in_last_t, 0)
n_in_last_t = np.clip(n_in_last_t, 0, 1)
masked = n_in_last_t - vid_y_gt
# 1: present before/after, but not in current
# 0: present before/after and in current, or not present before/after and not in current
# -1: not present before/after and in current
masked = np.clip(masked, 0, 1)
y_gt_mask.append(masked)
y_gt = np.concatenate(y_gt, 0).astype(np.uint8)
y_gt_mask = np.concatenate(y_gt_mask, 0).astype(np.uint8)
return conditional_t(y_pred, y_gt, y_gt_mask, thresh, avg)
if __name__ == '__main__':
warnings.simplefilter("ignore")
gt_labels = [np.random.randint(0, 2, (100, 5)), np.random.randint(0, 2, (200, 5))]
pred_probs = [np.random.uniform(0, 1, (100, 5)), np.random.uniform(0, 1, (200, 5))]
prec0, re0, ns0, map0 = conditional_metric(pred_probs, gt_labels, t=0, avg=True)
fs0 = get_f1(prec0, re0) # action conditional f1-score
print('Precision(c_i|c_j,0)=', prec0)
print('Recall(c_i|c_j,0)=', re0)
print('F1Score(c_i|c_j,0)=', map0)
print('mAP(c_i|c_j,0)=', fs0)
print()
prec20, re20, ns20, map20 = conditional_metric(pred_probs, gt_labels, t=20, avg=True)
fs20 = get_f1(prec20, re20) # action conditional f1-score
print('Precision(c_i|c_j,20)=', prec20)
print('Recall(c_i|c_j,20)=', re20)
print('F1Score(c_i|c_j,20)=', map20)
print('mAP(c_i|c_j,20)=', fs20)