-
Notifications
You must be signed in to change notification settings - Fork 12
/
kappa_test.py
105 lines (92 loc) · 3.55 KB
/
kappa_test.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
'''
Tests the segmentation versions of Cohen's and Fleiss' Kappa.
.. moduleauthor:: Chris Fournier <chris.m.fournier@gmail.com>
'''
import unittest
from decimal import Decimal
from .kappa import fleiss_kappa_linear
from ..data.samples import (KAZANTSEVA2012_G5, KAZANTSEVA2012_G2,
COMPLETE_AGREEMENT, LARGE_DISAGREEMENT)
class TestKappa(unittest.TestCase):
'''
Test segmentation versions of Cohen's Kappa and Fleiss' Multi-Kappa.
'''
def test_fleiss_kappa_linear_g5(self):
'''
Test Kappa upon Group 5 of Kazantseva (2012) data.
'''
data = KAZANTSEVA2012_G5
self.assertEqual(fleiss_kappa_linear(data),
Decimal('0.2374030241245084051985691283'))
def test_fleiss_kappa_linear_g5_ch1(self):
'''
Test Kappa upon Group 5, Chapter 1, of Kazantseva (2012) data.
'''
data = KAZANTSEVA2012_G5['ch1']
data = {'ch1': data}
self.assertEqual(fleiss_kappa_linear(data),
Decimal('0.1940298507462686567164179104'))
def test_fleiss_kappa_linear_g2(self):
'''
Test Kappa upon Group 2 of Kazantseva (2012) data.
'''
data = KAZANTSEVA2012_G2
self.assertEqual(fleiss_kappa_linear(data),
Decimal('0.4068521352341765453201908783'))
def test_fleiss_kappa_linear_g2_ch2(self):
'''
Test Kappa upon Group 2, Chapter 2, of Kazantseva (2012) data.
'''
data = KAZANTSEVA2012_G2['ch2']
data = {'ch2': data}
self.assertEqual(fleiss_kappa_linear(data),
Decimal('0.5197205281147376181221097781'))
def test_fleiss_kappa_disagree(self):
'''
Test Kappa upon a hypothetical dataset containing large disagreement.
'''
data = LARGE_DISAGREEMENT
self.assertEqual(fleiss_kappa_linear(data),
Decimal('-0.05952156715012243360331512524'))
def test_fleiss_kappa(self):
'''
Test Cohen's and Fleiss' Kappa.
'''
data1 = {'i1': {'c1': [2, 8, 2, 1],
'c2': [2, 1, 7, 2, 1]}}
kappa1 = fleiss_kappa_linear(data1)
kappa1f = fleiss_kappa_linear(data1)
self.assertEqual(kappa1,
Decimal('0.7096774193548387096774193548'))
self.assertEqual(kappa1, kappa1f)
data2 = {'i1': {'c1': [2, 8, 2, 1],
'c2': [11, 2]}}
kappa2 = fleiss_kappa_linear(data2)
kappa2f = fleiss_kappa_linear(data2)
self.assertEqual(kappa2,
Decimal('0.1176470588235294117647058823'))
self.assertEqual(kappa2, kappa2f)
self.assertTrue(kappa2 < kappa1)
def test_fleiss_kappa_complete(self):
'''
Test Kappa upon a hypothetical dataset containing complete agreement.
'''
data_complete = COMPLETE_AGREEMENT
kappa = fleiss_kappa_linear(data_complete)
self.assertEqual(kappa, Decimal('1.0'))
def test_exception_coders(self):
'''
Test exception.
'''
data = {'i1': {'c1': [2, 8, 2, 1]}}
self.assertRaises(Exception, fleiss_kappa_linear, data)
def test_exception_items(self):
'''
Test exception.
'''
data = {'i1': {'c1': [2, 8, 2, 1],
'c2': [2, 8, 2, 1]},
'i2': {'c1': [2, 1, 7, 2, 1],
'c2': [2, 8, 2, 1],
'c3': [2, 8, 2, 1]}}
self.assertRaises(Exception, fleiss_kappa_linear, data)