-
Notifications
You must be signed in to change notification settings - Fork 4
/
evaluation.py
221 lines (173 loc) · 6.84 KB
/
evaluation.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
from __future__ import print_function
import os
import sys
import csv
import pickle
import numpy
import time
import numpy as np
import torch
from collections import OrderedDict
from utils.common import makedirsforfile, checkToSkip
from simpleknn.bigfile import BigFile
class AverageMeter(object):
"""Computes and stores the average and current value"""
def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=0):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / (.0001 + self.count)
def __str__(self):
"""String representation for logging
"""
# for values that should be recorded exactly e.g. iteration number
if self.count == 0:
return str(self.val)
# for stats
return '%.4f (%.4f)' % (self.val, self.avg)
class LogCollector(object):
"""A collection of logging objects that can change from train to val"""
def __init__(self):
# to keep the order of logged variables deterministic
self.meters = OrderedDict()
def update(self, k, v, n=0):
# create a new meter if previously not recorded
if k not in self.meters:
self.meters[k] = AverageMeter()
self.meters[k].update(v, n)
def __str__(self):
"""Concatenate the meters in one log line
"""
s = ''
for i, (k, v) in enumerate(self.meters.iteritems()):
if i > 0:
s += ' '
s += k + ' ' + str(v)
return s
def tb_log(self, tb_logger, prefix='', step=None):
"""Log using tensorboard
"""
for k, v in self.meters.iteritems():
tb_logger.log_value(prefix + k, v.val, step=step)
def encode_data(model, data_loader, log_step=10, logging=print):
"""Encode all images and captions loadable by `data_loader`
"""
batch_time = AverageMeter()
# switch to evaluate mode
model.val_start()
end = time.time()
# numpy array to keep all the embeddings
video_embs = None
video_ids_list = []
for i, (videos, ids, idxs) in enumerate(data_loader):
# compute the embeddings
video_emb = model.forward_emb(videos, volatile=True)
# initialize the numpy arrays given the size of the embeddings
if video_embs is None:
video_embs = np.zeros((len(data_loader.dataset), video_emb.size(1)))
# preserve the embeddings by copying from gpu and converting to numpy
video_embs[list(idxs)] = video_emb.data.cpu().numpy().copy()
video_ids_list.extend(ids)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % log_step == 0:
logging('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
.format(
i, len(data_loader), batch_time=batch_time))
del videos
return video_embs, video_ids_list
def cal_rel_index(rele_file):
with open(rele_file,'r') as csvfile:
data=[]
csv_reader=csv.reader(csvfile)
for line in csv_reader:
data.append(line)
return data
def re_cal_scores(scores, rel_index, n, sumofdata):
scores_list = []
for i, index in enumerate(rel_index):
data = []
data.append(scores[i])
if i < sumofdata:
for j in index[:n]:
data.append(scores[int(j)])
scores_list.append(sum(data) / len(data))
else:
scores_list.append(data)
# test video has no available relations
scores_list.append(scores[len(rel_index): len(scores)])
return scores_list
def score2result(scores, test_video_list, cand_video_list, rel_index, n):
video2predrank = {}
n_rows, n_column = scores.shape
assert n_rows == len(test_video_list)
assert n_column == len(cand_video_list)
for i, test_video in enumerate(test_video_list):
score_list = scores[i]
if rel_index is not None:
sumofdata = n_column - n_rows
score_list = re_cal_scores(score_list, rel_index, n, sumofdata)
cand_video_score_list = zip(cand_video_list, score_list)
sorted_cand_video_score_list = sorted(cand_video_score_list, key=lambda v:v[1], reverse=True)
#video2predrank[test_video] = [x[0] for x in sorted_cand_video_score_list]
predrank = [x[0] for x in sorted_cand_video_score_list]
predrank.remove(test_video)
video2predrank[test_video] = predrank
return video2predrank
def score2result_fusion(scores, test_video_list, cand_video_list):
video2predrank = {}
n_rows, n_column = scores.shape
assert n_rows == len(test_video_list)
assert n_column == len(cand_video_list)
for i, test_video in enumerate(test_video_list):
score_list = scores[i]
cand_video_score_list = zip(cand_video_list, score_list)
sorted_cand_video_score_list = sorted(cand_video_score_list, key=lambda v:v[1], reverse=True)
predrank = [x[0] for x in sorted_cand_video_score_list]
predrank.remove(test_video)
video2predrank[test_video] = predrank
return video2predrank
def do_predict(test_video_emd, test_video_list, cand_video_emd, cand_video_list, rel_index=None, n=5, output_dir=None, overwrite=0, no_imgnorm=False):
if no_imgnorm:
scores = cal_score(test_video_emd, cand_video_emd, measure='cosine')
else:
scores = cal_score(test_video_emd, cand_video_emd, measure='dot')
video2predrank = score2result(scores, test_video_list, cand_video_list, rel_index, n)
if output_dir is not None:
output_file = os.path.join(output_dir, 'pred_scores_matrix.pth.tar')
if checkToSkip(output_file, overwrite):
sys.exit(0)
makedirsforfile(output_file)
torch.save({'scores': scores, 'test_videos': test_video_list, 'cand_videos': cand_video_list}, output_file)
print("write score matrix into: " + output_file)
return video2predrank
# def cal_error(images, captions, measure='cosine', n_caption=2):
# """
# Images->Text (Image Annotation)
# Images: (5N, K) matrix of images
# Captions: (5N, K) matrix of captions
# """
# idx = range(0, images.shape[0], n_caption)
# im = images[idx, :]
# if measure == 'cosine':
# errors = -1*numpy.dot(captions, im.T)
# return errors
def cal_score(video_1, video_2, measure='cosine'):
if measure == 'cosine':
# l2 normalization
import sklearn.preprocessing as preprocessing
video_1 = preprocessing.normalize(video_1, norm='l2')
video_2 = preprocessing.normalize(video_2, norm='l2')
scores = numpy.dot(video_1, video_2.T)
elif measure == 'dot':
scores = numpy.dot(video_1, video_2.T)
return scores