This repository has been archived by the owner on Jan 3, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 2
/
evaluate.R
142 lines (120 loc) · 7.3 KB
/
evaluate.R
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
################################################################################
# HACKATHON PARTICIPANTS -- DO NOT EDIT THIS FILE #
################################################################################
VERSION <- commandArgs(trailingOnly = TRUE)[1]
USE_TESTING_DATA <- file.exists("./csvs/testing.csv")
################################################################################
# import predictions
predicted_mortality_training <- read.table("./output/predicted_mortality_training.dat")[[1]]
predicted_fss_training <- read.table("./output/predicted_fss_training.dat")[[1]]
raw_data_training <- read.csv("./csvs/training.csv")
if (USE_TESTING_DATA) {
predicted_mortality_testing <- read.table("./output/predicted_mortality_testing.dat")[[1]]
predicted_fss_testing <- read.table("./output/predicted_fss_testing.dat")[[1]]
raw_data_testing <- read.csv("./csvs/testing.csv")
}
################################################################################
# Get the observed outcomes
observed_fss_training <-
Reduce(function(x, y) x + y,
x = raw_data_training[raw_data_training$hospdisposition != "Mortality", grep("^fss", names(raw_data_training))])
observed_fss_training <- na.omit(observed_fss_training)
attributes(observed_fss_training) <- NULL
observed_mortality_training <- ifelse(raw_data_training$hospdisposition == "Mortality", "Mortality", "Alive")
if (USE_TESTING_DATA) {
observed_fss_testing <-
Reduce(function(x, y) x + y,
x = raw_data_testing[raw_data_testing$hospdisposition != "Mortality", grep("^fss", names(raw_data_testing))])
observed_fss_testing <- na.omit(observed_fss_testing)
attributes(observed_fss_testing) <- NULL
observed_mortality_testing <- ifelse(raw_data_testing$hospdisposition == "Mortality", "Mortality", "Alive")
}
################################################################################
# Checks:
stopifnot(length(observed_mortality_training) == length(predicted_mortality_training))
stopifnot(length(observed_fss_training) == length(predicted_fss_training))
stopifnot(all(predicted_mortality_training %in% c("Mortality", "Alive")))
stopifnot(all(predicted_fss_training >= 6) & all(predicted_fss_training <= 30))
stopifnot(all(predicted_fss_training %% 1 == 0))
if (USE_TESTING_DATA) {
stopifnot(length(observed_mortality_testing) == length(predicted_mortality_testing))
stopifnot(length(observed_fss_testing) == length(predicted_fss_testing))
stopifnot(all(predicted_mortality_testing %in% c("Mortality", "Alive")))
stopifnot(all(predicted_fss_testing >= 6) & all(predicted_fss_testing <= 30))
stopifnot(all(predicted_fss_testing %% 1 == 0))
}
################################################################################
# Define a function for writing out assessment results
report_assessment <- function(label, value) {
cat(label, "|", value, "\n", file = "./output/evaluation.txt", sep = " ", append = TRUE)
}
################################################################################
# Mortality Assessments
mortality_assessments <- function(observed, predicted) {
TP <- sum(observed == "Mortality" & predicted == "Mortality")
TN <- sum(observed == "Alive" & predicted == "Alive")
FP <- sum(observed == "Alive" & predicted == "Mortality")
FN <- sum(observed == "Mortality" & predicted == "Alive")
rtn <- list()
rtn$ACCURACY <- (TP + TN) / (TP + TN + FP + FN)
rtn$SENSITIVITY <- TP / (TP + FN)
rtn$SPECIFICITY <- TN / (TN + FP)
rtn$PPV <- TP / (TP + FP)
rtn$NPV <- TN / (TN + FN)
rtn$MCC <- (TP * TN - FP * FN) / sqrt( (TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))
rtn$F1 <- (2 * TP) / (2 * TP + FP + FN)
rtn
}
ma_training <- mortality_assessments(observed_mortality_training, predicted_mortality_training)
report_assessment("Accuracy for mortality model on training data", ma_training$ACCURACY)
report_assessment("Sensitivity for mortality model on training data", ma_training$SENSITIVITY)
report_assessment("Specificity for mortality model on training data", ma_training$SPECIFICITY)
report_assessment("PPV for mortality model on training data", ma_training$PPV)
report_assessment("NPV for mortality model on training data", ma_training$NPV)
report_assessment("MCC for mortality model on training data", ma_training$MCC)
report_assessment("F1 for mortality model on training data", ma_training$F1)
if (USE_TESTING_DATA) {
ma_testing <- mortality_assessments(observed_mortality_testing, predicted_mortality_testing)
if (isTRUE(grepl("^v1\\.\\d", VERSION))) {
report_assessment("Accuracy for mortality model on testing data", ma_testing$ACCURACY)
report_assessment("Sensitivity for mortality model on testing data", ma_testing$SENSITIVITY)
report_assessment("Specificity for mortality model on testing data", ma_testing$SPECIFICITY)
report_assessment("PPV for mortality model on testing data", ma_testing$PPV)
report_assessment("NPV for mortality model on testing data", ma_testing$NPV)
report_assessment("MCC for mortality model on testing data", ma_testing$MCC)
report_assessment("F1 for mortality model on testing data", ma_testing$F1)
} else {
cat("Mortality Assessments on testing data | SUCCESS\n", file = "./output/evaluation.txt", append = TRUE)
}
}
################################################################################
# Assessment for FSS
MSE <- mean((observed_fss_training - predicted_fss_training)^2)
MIN_AD <- min(abs(observed_fss_training - predicted_fss_training))
MEDIAN_AD <- median(abs(observed_fss_training - predicted_fss_training))
MEAN_AD <- mean(abs(observed_fss_training - predicted_fss_training))
MAX_AD <- max(abs(observed_fss_training - predicted_fss_training))
report_assessment("MSE for predicted FSS on training data", MSE)
report_assessment("Min abs difference in FSS prediction and observed on training data", MIN_AD)
report_assessment("Median abs difference in FSS prediction and observed on training data", MEDIAN_AD)
report_assessment("Mean abs difference in FSS prediction and observed on training data", MEAN_AD)
report_assessment("Max abs difference in FSS prediction and observed on training data", MAX_AD)
if (USE_TESTING_DATA) {
MSE <- mean((observed_fss_testing - predicted_fss_testing)^2)
MIN_AD <- min(abs(observed_fss_testing - predicted_fss_testing))
MEDIAN_AD <- median(abs(observed_fss_testing - predicted_fss_testing))
MEAN_AD <- mean(abs(observed_fss_testing - predicted_fss_testing))
MAX_AD <- max(abs(observed_fss_testing - predicted_fss_testing))
if (isTRUE(grepl("^v1\\.\\d", VERSION))) {
report_assessment("MSE for predicted FSS on testing data", MSE)
report_assessment("Min abs difference in FSS prediction and observed on testing data", MIN_AD)
report_assessment("Median abs difference in FSS prediction and observed on testing data", MEDIAN_AD)
report_assessment("Mean abs difference in FSS prediction and observed on testing data", MEAN_AD)
report_assessment("Max abs difference in FSS prediction and observed on testing data", MAX_AD)
} else {
cat("FSS Assessments on testing data | SUCCESS\n", file = "./output/evaluation.txt", append = TRUE)
}
}
################################################################################
# End of File #
################################################################################