/
eval.R
118 lines (101 loc) · 3.61 KB
/
eval.R
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
########################################################
set.seed(777)
actual = c('a','b','c')[runif(100, 1,4)]
predicted = actual
predicted[runif(30,1,100)] = actual[runif(30,1,100)]
results = Evaluate(actual=actual, predicted=predicted)
########################################################
#input actual & predicted vectors or actual vs predicted confusion matrix
Evaluate = function(actual=NULL, predicted=NULL, cm=NULL){
if(is.null(cm)) {
naVals = union(which(is.na(actual)), which(is.na(predicted)))
if(length(naVals) > 0) {
actual = actual[-naVals]
predicted = predicted[-naVals]
}
f = factor(union(unique(actual), unique(predicted)))
actual = factor(actual, levels = levels(f))
predicted = factor(predicted, levels = levels(f))
cm = as.matrix(table(Actual=actual, Predicted=predicted))
}
n = sum(cm) # number of instances
nc = nrow(cm) # number of classes
diag = diag(cm) # number of correctly classified instances per class
rowsums = apply(cm, 1, sum) # number of instances per class
colsums = apply(cm, 2, sum) # number of predictions per class
p = rowsums / n # distribution of instances over the classes
q = colsums / n # distribution of instances over the predicted classes
#accuracy
accuracy = sum(diag) / n
#per class prf
recall = diag / rowsums
precision = diag / colsums
f1 = 2 * precision * recall / (precision + recall)
#macro prf
macroPrecision = mean(precision)
macroRecall = mean(recall)
macroF1 = mean(f1)
#1-vs-all matrix
oneVsAll = lapply(1 : nc,
function(i){
v = c(cm[i,i],
rowsums[i] - cm[i,i],
colsums[i] - cm[i,i],
n-rowsums[i] - colsums[i] + cm[i,i]);
return(matrix(v, nrow = 2, byrow = T))})
s = matrix(0, nrow=2, ncol=2)
for(i in 1:nc){s=s+oneVsAll[[i]]}
#avg accuracy
avgAccuracy = sum(diag(s))/sum(s)
#micro prf
microPrf = (diag(s) / apply(s,1, sum))[1];
#majority class
mcIndex = which(rowsums==max(rowsums))[1] # majority-class index
mcAccuracy = as.numeric(p[mcIndex])
mcRecall = 0*p; mcRecall[mcIndex] = 1
mcPrecision = 0*p; mcPrecision[mcIndex] = p[mcIndex]
mcF1 = 0*p; mcF1[mcIndex] = 2 * mcPrecision[mcIndex] / (mcPrecision[mcIndex] + 1)
#random/expected accuracy
expAccuracy = sum(p*q)
#kappa
kappa = (accuracy - expAccuracy) / (1 - expAccuracy)
#random guess
rgAccuracy = 1 / nc
rgPrecision = p
rgRecall = 0*p + 1 / nc
rgF1 = 2 * p / (nc * p + 1)
#random weighted guess
rwgAccurcy = sum(p^2)
rwgPrecision = p
rwgRecall = p
rwgF1 = p
classNames = names(diag)
if(is.null(classNames)) classNames = paste("C",(1:nc),sep="")
metrics = rbind(
Accuracy = accuracy,
Precision = precision,
Recall = recall,
F1 = f1,
MacroAvgPrecision = macroPrecision,
MacroAvgRecall = macroRecall,
MacroAvgF1 = macroF1,
AvgAccuracy = avgAccuracy,
MicroAvgPrecision = microPrf,
MicroAvgRecall = microPrf,
MicroAvgF1 = microPrf,
MajorityClassAccuracy = mcAccuracy,
MajorityClassPrecision = mcPrecision,
MajorityClassRecall = mcRecall,
MajorityClassF1 = mcF1,
Kappa = kappa,
RandomGuessAccuracy = rgAccuracy,
RandomGuessPrecision = rgPrecision,
RandomGuessRecall = rgRecall,
RandomGuessF1 = rgF1,
RandomWeightedGuessAccuracy = rwgAccurcy,
RandomWeightedGuessPrecision = rwgPrecision,
RandomWeightedGuessRecall = rwgRecall,
RandomWeightedGuessF1 = rwgF1)
colnames(metrics) = classNames
return(list(ConfusionMatrix = cm, Metrics = metrics))
}