-
Notifications
You must be signed in to change notification settings - Fork 0
/
evaluation.py
60 lines (51 loc) · 2.16 KB
/
evaluation.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
import numpy as np
from sklearn.metrics import roc_auc_score
from rpy2.robjects import r
import rpy2.robjects as robjects
from rpy2.robjects import pandas2ri
pandas2ri.activate()
def ici(labels, predictions):
"""Function to compute the integrated calibration index for a binary outcome.
Smooth calibration curves based on local polynomial regression is produced regressing the binary outcome with the
predicted risk. Then the ICI is the weighted difference between smoothed observed proportions and predicted probs.
Reference: Austin and Steyerberg (2019). https://doi.org/10.1002/sim.8281
Input
- labels: observed binary outcomes
- predictions: predicted probabilities
Output
- ici: integrated calibration index
"""
# Converting arrays to R
y, p = robjects.FloatVector(labels), robjects.FloatVector(predictions)
df = robjects.DataFrame({"P": p, "Y": y})
# Fitting the model and making predictions
loess_fit = r.loess("Y ~ P", data=df)
p_calibrate = np.array(r.predict(loess_fit, newdata=p))
# Computing ICI
ici_ = np.mean(np.abs(p_calibrate-predictions))
return ici_
class MetricsCV:
"""Class to keep track of AUC and ICI in all the different cross-validation trainings.
"""
def __init__(self, path_results, set_name):
"""
Input:
- path_results (str): path where results have to be saved
- set_name (str): "train", "val" or "test"
"""
self.auc = []
self.ici = []
self.path_results = path_results
self.set_name = set_name
def compute_metrics(self, labels, predictions):
"""Compute metrics for a given set of labels/predictions and append it to the values of the previous folds
Input:
- labels (array-like): true labels
- predictions (array-like): target scores
"""
self.auc.append(roc_auc_score(labels, predictions))
self.ici.append(ici(labels, predictions))
def save_results(self):
"""Save results as numpy arrays"""
np.save(self.path_results + 'auc_' + self.set_name, self.auc)
np.save(self.path_results + 'ici_' + self.set_name, self.ici)