-
Notifications
You must be signed in to change notification settings - Fork 8
/
example_kappa.py
86 lines (79 loc) · 1.8 KB
/
example_kappa.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
from fleiss import fleissKappa
if __name__ == "__main__":
print("Example run to calculate Fleiss' Kappa")
print("\nTest case 1 - Fleiss 1971")
#Fleiss, 1971 example
rate = \
[
[0,0,0,6,0],
[0,3,0,0,3],
[0,1,4,0,1],
[0,0,0,0,6],
[0,3,0,3,0],
[2,0,4,0,0],
[0,0,4,0,2],
[2,0,3,1,0],
[2,0,0,4,0],
[0,0,0,0,6],
[1,0,0,5,0],
[1,1,0,4,0],
[0,3,3,0,0],
[1,0,0,5,0],
[0,2,0,3,1],
[0,0,5,0,1],
[3,0,0,1,2],
[5,1,0,0,0],
[0,2,0,4,0],
[1,0,2,0,3],
[0,0,0,0,6],
[0,1,0,5,0],
[0,2,0,1,3],
[2,0,0,4,0],
[1,0,0,4,1],
[0,5,0,1,0],
[4,0,0,0,2],
[0,2,0,4,0],
[1,0,5,0,0],
[0,0,0,0,6]
]
kappa = fleissKappa(rate,6)
assert(kappa==0.43)
print("\nTest case 2 - Wikipedia example")
#wikipedia example
rate = \
[
[0,0,0,0,14],
[0,2,6,4,2],
[0,0,3,5,6],
[0,3,9,2,0],
[2,2,8,1,1],
[7,7,0,0,0],
[3,2,6,3,0],
[2,5,3,2,2],
[6,5,2,1,0],
[0,2,2,3,7]
]
kappa = fleissKappa(rate,14)
assert(kappa==0.21)
print("\nTest case 3 - perfect agreement")
#kappa = 1
rate = \
[
[2,0,0,0],
[0,2,0,0],
[0,0,2,0],
[0,0,0,2]
]
kappa = fleissKappa(rate,2)
assert(kappa==1)
print("\nTest case 4 - perfect expected agreement")
#kappa = -inf (not defined as divide by 0)
rate = \
[
[2,0,0,0],
[2,0,0,0],
[2,0,0,0],
[2,0,0,0]
]
kappa = fleissKappa(rate,2)
assert(kappa==-float("inf"))