-
Notifications
You must be signed in to change notification settings - Fork 268
/
recover_best_model.py
89 lines (67 loc) · 2.9 KB
/
recover_best_model.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
def recover_best_model(x_train,
y_train,
x_val,
y_val,
experiment_log,
input_model,
x_cross=None,
y_cross=None,
n_models=5,
task='multi_label'):
'''Recover best models from Talos experiment log.
x_train | array | same as was used in the experiment
y_train | array | same as was used in the experiment
x_val | array | same as was used in the experiment
y_val | array | same as was used in the experiment
x_cross | array | data for the cross-validation or None for use x_val
y_cross | array | data for the cross-validation or None for use y_val
experiment_log | str | path to the Talos experiment log
input_model | function | model used in the experiment
n_models | int | number of models to cross-validate
task | str | binary, multi_class, multi_label or continuous
Returns a pandas dataframe with the cross-validation results
and the models.
'''
import pandas as pd
import sklearn as sk
import numpy as np
from talos.utils.validation_split import kfold
# read the experiment log into a dataframe
df = pd.read_csv(experiment_log)
# handle input data scenarios
if x_cross is None or y_cross is None:
x_cross = x_val
y_cross = y_val
# for final output
results = []
models = []
for i in range(n_models):
# get the params for the model and train it
params = df.sort_values('val_acc', ascending=False).drop('val_acc', 1).iloc[i].to_dict()
history, model = input_model(x_train, y_train, x_val, y_val, params)
# start kfold cross-validation
out = []
folds = 5
kx, ky = kfold(x_cross, y_cross, folds, True)
for i in range(folds):
y_pred = model.predict(kx[i]).argmax(axis=1)
if task == 'binary':
y_pred = y_pred >= .5
scores = sk.metrics.f1_score(y_pred, ky[i], average='binary')
elif task == 'multi_class':
y_pred = y_pred.argmax(axis=-1)
scores = sk.metrics.f1_score(y_pred, ky[i], average='macro')
if task == 'multi_label':
y_pred = model.predict(kx[i]).argmax(axis=1)
scores = sk.metrics.f1_score(y_pred,
ky[i].argmax(axis=1),
average='macro')
elif task == 'continuous':
y_pred = model.predict(kx[i])
scores = sk.metrics.mean_absolute_error(y_pred, ky[i])
out.append(scores)
results.append(np.mean(out))
models.append(model)
out = df.sort_values('val_acc', ascending=False).head(n_models)
out['crossval_mean_f1score'] = results
return out, models