-
-
Notifications
You must be signed in to change notification settings - Fork 122
/
warning_test.py
549 lines (549 loc) · 51.4 KB
/
warning_test.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
# -*- coding: utf-8 -*-
"""
>>> from pycm import *
>>> from pytest import warns
>>> large_cm = ConfusionMatrix(list(range(10))+[2,3,5],list(range(10))+[1,7,2])
>>> with warns(RuntimeWarning, match='The confusion matrix is a high dimension matrix'):
... large_cm.print_matrix()
Predict 0 1 2 3 4 5 6 7 8 9
Actual
0 1 0 0 0 0 0 0 0 0 0
<BLANKLINE>
1 0 1 0 0 0 0 0 0 0 0
<BLANKLINE>
2 0 1 1 0 0 0 0 0 0 0
<BLANKLINE>
3 0 0 0 1 0 0 0 1 0 0
<BLANKLINE>
4 0 0 0 0 1 0 0 0 0 0
<BLANKLINE>
5 0 0 1 0 0 1 0 0 0 0
<BLANKLINE>
6 0 0 0 0 0 0 1 0 0 0
<BLANKLINE>
7 0 0 0 0 0 0 0 1 0 0
<BLANKLINE>
8 0 0 0 0 0 0 0 0 1 0
<BLANKLINE>
9 0 0 0 0 0 0 0 0 0 1
<BLANKLINE>
>>> with warns(RuntimeWarning, match='The confusion matrix is a high dimension matrix'):
... large_cm.print_normalized_matrix()
Predict 0 1 2 3 4 5 6 7 8 9
Actual
0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
<BLANKLINE>
1 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
<BLANKLINE>
2 0.0 0.5 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0
<BLANKLINE>
3 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.5 0.0 0.0
<BLANKLINE>
4 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0
<BLANKLINE>
5 0.0 0.0 0.5 0.0 0.0 0.5 0.0 0.0 0.0 0.0
<BLANKLINE>
6 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0
<BLANKLINE>
7 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0
<BLANKLINE>
8 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0
<BLANKLINE>
9 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0
<BLANKLINE>
>>> with warns(RuntimeWarning, match='The confusion matrix is a high dimension matrix'):
... large_cm.stat()
Overall Statistics :
<BLANKLINE>
95% CI (0.5402,0.99827)
ACC Macro 0.95385
ARI -0.04
AUNP 0.87121
AUNU 0.91212
Bangdiwala B 0.58824
Bennett S 0.74359
CBA 0.75
CSI 0.7
Chi-Squared 91.0
Chi-Squared DF 81
Conditional Entropy 0.46154
Cramer V 0.88192
Cross Entropy 3.39275
F1 Macro 0.81667
F1 Micro 0.76923
FNR Macro 0.15
FNR Micro 0.23077
FPR Macro 0.02576
FPR Micro 0.02564
Gwet AC1 0.7438
Hamming Loss 0.23077
Joint Entropy 3.70044
KL Divergence 0.15385
Kappa 0.74342
Kappa 95% CI (0.48877,0.99807)
Kappa No Prevalence 0.53846
Kappa Standard Error 0.12992
Kappa Unbiased 0.74172
Krippendorff Alpha 0.75166
Lambda A 0.72727
Lambda B 0.72727
Mutual Information 2.77736
NIR 0.15385
Overall ACC 0.76923
Overall CEN 0.09537
Overall J (7.33333,0.73333)
Overall MCC 0.75333
Overall MCEN 0.10746
Overall RACC 0.10059
Overall RACCU 0.10651
P-Value 0.0
PPV Macro 0.85
PPV Micro 0.76923
Pearson C 0.93541
Phi-Squared 7.0
RCI 0.8575
RR 1.3
Reference Entropy 3.2389
Response Entropy 3.2389
SOA1(Landis & Koch) Substantial
SOA2(Fleiss) Intermediate to Good
SOA3(Altman) Good
SOA4(Cicchetti) Excellent
SOA5(Cramer) Very Strong
SOA6(Matthews) Strong
Scott PI 0.74172
Standard Error 0.11685
TNR Macro 0.97424
TNR Micro 0.97436
TPR Macro 0.85
TPR Micro 0.76923
Zero-one Loss 3
<BLANKLINE>
Class Statistics :
<BLANKLINE>
Classes 0 1 2 3 4 5 6 7 8 9
ACC(Accuracy) 1.0 0.92308 0.84615 0.92308 1.0 0.92308 1.0 0.92308 1.0 1.0
AGF(Adjusted F-score) 1.0 0.90468 0.6742 0.71965 1.0 0.71965 1.0 0.90468 1.0 1.0
AGM(Adjusted geometric mean) 1.0 0.93786 0.78186 0.84135 1.0 0.84135 1.0 0.93786 1.0 1.0
AM(Difference between automatic and manual classification) 0 1 0 -1 0 -1 0 1 0 0
AUC(Area under the ROC curve) 1.0 0.95833 0.70455 0.75 1.0 0.75 1.0 0.95833 1.0 1.0
AUCI(AUC value interpretation) Excellent Excellent Good Good Excellent Good Excellent Excellent Excellent Excellent
AUPR(Area under the PR curve) 1.0 0.75 0.5 0.75 1.0 0.75 1.0 0.75 1.0 1.0
BB(Braun-Blanquet similarity) 1.0 0.5 0.5 0.5 1.0 0.5 1.0 0.5 1.0 1.0
BCD(Bray-Curtis dissimilarity) 0.0 0.03846 0.0 0.03846 0.0 0.03846 0.0 0.03846 0.0 0.0
BM(Informedness or bookmaker informedness) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0
CEN(Confusion entropy) 0 0.1267 0.23981 0.1267 0 0.1267 0 0.1267 0 0
DOR(Diagnostic odds ratio) None None 10.0 None None None None None None None
DP(Discriminant power) None None 0.55133 None None None None None None None
DPI(Discriminant power interpretation) None None Poor None None None None None None None
ERR(Error rate) 0.0 0.07692 0.15385 0.07692 0.0 0.07692 0.0 0.07692 0.0 0.0
F0.5(F0.5 score) 1.0 0.55556 0.5 0.83333 1.0 0.83333 1.0 0.55556 1.0 1.0
F1(F1 score - harmonic mean of precision and sensitivity) 1.0 0.66667 0.5 0.66667 1.0 0.66667 1.0 0.66667 1.0 1.0
F2(F2 score) 1.0 0.83333 0.5 0.55556 1.0 0.55556 1.0 0.83333 1.0 1.0
FDR(False discovery rate) 0.0 0.5 0.5 0.0 0.0 0.0 0.0 0.5 0.0 0.0
FN(False negative/miss/type 2 error) 0 0 1 1 0 1 0 0 0 0
FNR(Miss rate or false negative rate) 0.0 0.0 0.5 0.5 0.0 0.5 0.0 0.0 0.0 0.0
FOR(False omission rate) 0.0 0.0 0.09091 0.08333 0.0 0.08333 0.0 0.0 0.0 0.0
FP(False positive/type 1 error/false alarm) 0 1 1 0 0 0 0 1 0 0
FPR(Fall-out or false positive rate) 0.0 0.08333 0.09091 0.0 0.0 0.0 0.0 0.08333 0.0 0.0
G(G-measure geometric mean of precision and sensitivity) 1.0 0.70711 0.5 0.70711 1.0 0.70711 1.0 0.70711 1.0 1.0
GI(Gini index) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0
GM(G-mean geometric mean of specificity and sensitivity) 1.0 0.95743 0.6742 0.70711 1.0 0.70711 1.0 0.95743 1.0 1.0
HD(Hamming distance) 0 1 2 1 0 1 0 1 0 0
IBA(Index of balanced accuracy) 1.0 0.99306 0.2686 0.25 1.0 0.25 1.0 0.99306 1.0 1.0
ICSI(Individual classification success index) 1.0 0.5 0.0 0.5 1.0 0.5 1.0 0.5 1.0 1.0
IS(Information score) 3.70044 2.70044 1.70044 2.70044 3.70044 2.70044 3.70044 2.70044 3.70044 3.70044
J(Jaccard index) 1.0 0.5 0.33333 0.5 1.0 0.5 1.0 0.5 1.0 1.0
LS(Lift score) 13.0 6.5 3.25 6.5 13.0 6.5 13.0 6.5 13.0 13.0
MCC(Matthews correlation coefficient) 1.0 0.677 0.40909 0.677 1.0 0.677 1.0 0.677 1.0 1.0
MCCI(Matthews correlation coefficient interpretation) Very Strong Moderate Weak Moderate Very Strong Moderate Very Strong Moderate Very Strong Very Strong
MCEN(Modified confusion entropy) 0 0.11991 0.2534 0.11991 0 0.11991 0 0.11991 0 0
MK(Markedness) 1.0 0.5 0.40909 0.91667 1.0 0.91667 1.0 0.5 1.0 1.0
N(Condition negative) 12 12 11 11 12 11 12 12 12 12
NLR(Negative likelihood ratio) 0.0 0.0 0.55 0.5 0.0 0.5 0.0 0.0 0.0 0.0
NLRI(Negative likelihood ratio interpretation) Good Good Negligible Negligible Good Negligible Good Good Good Good
NPV(Negative predictive value) 1.0 1.0 0.90909 0.91667 1.0 0.91667 1.0 1.0 1.0 1.0
OC(Overlap coefficient) 1.0 1.0 0.5 1.0 1.0 1.0 1.0 1.0 1.0 1.0
OOC(Otsuka-Ochiai coefficient) 1.0 0.70711 0.5 0.70711 1.0 0.70711 1.0 0.70711 1.0 1.0
OP(Optimized precision) 1.0 0.8796 0.55583 0.58974 1.0 0.58974 1.0 0.8796 1.0 1.0
P(Condition positive or support) 1 1 2 2 1 2 1 1 1 1
PLR(Positive likelihood ratio) None 12.0 5.5 None None None None 12.0 None None
PLRI(Positive likelihood ratio interpretation) None Good Fair None None None None Good None None
POP(Population) 13 13 13 13 13 13 13 13 13 13
PPV(Precision or positive predictive value) 1.0 0.5 0.5 1.0 1.0 1.0 1.0 0.5 1.0 1.0
PRE(Prevalence) 0.07692 0.07692 0.15385 0.15385 0.07692 0.15385 0.07692 0.07692 0.07692 0.07692
Q(Yule Q - coefficient of colligation) None None 0.81818 None None None None None None None
QI(Yule Q interpretation) None None Strong None None None None None None None
RACC(Random accuracy) 0.00592 0.01183 0.02367 0.01183 0.00592 0.01183 0.00592 0.01183 0.00592 0.00592
RACCU(Random accuracy unbiased) 0.00592 0.01331 0.02367 0.01331 0.00592 0.01331 0.00592 0.01331 0.00592 0.00592
TN(True negative/correct rejection) 12 11 10 11 12 11 12 11 12 12
TNR(Specificity or true negative rate) 1.0 0.91667 0.90909 1.0 1.0 1.0 1.0 0.91667 1.0 1.0
TON(Test outcome negative) 12 11 11 12 12 12 12 11 12 12
TOP(Test outcome positive) 1 2 2 1 1 1 1 2 1 1
TP(True positive/hit) 1 1 1 1 1 1 1 1 1 1
TPR(Sensitivity, recall, hit rate, or true positive rate) 1.0 1.0 0.5 0.5 1.0 0.5 1.0 1.0 1.0 1.0
Y(Youden index) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0
dInd(Distance index) 0.0 0.08333 0.5082 0.5 0.0 0.5 0.0 0.08333 0.0 0.0
sInd(Similarity index) 1.0 0.94107 0.64065 0.64645 1.0 0.64645 1.0 0.94107 1.0 1.0
<BLANKLINE>
>>> with warns(RuntimeWarning, match='The confusion matrix is a high dimension matrix'):
... print(large_cm)
Predict 0 1 2 3 4 5 6 7 8 9
Actual
0 1 0 0 0 0 0 0 0 0 0
<BLANKLINE>
1 0 1 0 0 0 0 0 0 0 0
<BLANKLINE>
2 0 1 1 0 0 0 0 0 0 0
<BLANKLINE>
3 0 0 0 1 0 0 0 1 0 0
<BLANKLINE>
4 0 0 0 0 1 0 0 0 0 0
<BLANKLINE>
5 0 0 1 0 0 1 0 0 0 0
<BLANKLINE>
6 0 0 0 0 0 0 1 0 0 0
<BLANKLINE>
7 0 0 0 0 0 0 0 1 0 0
<BLANKLINE>
8 0 0 0 0 0 0 0 0 1 0
<BLANKLINE>
9 0 0 0 0 0 0 0 0 0 1
<BLANKLINE>
<BLANKLINE>
<BLANKLINE>
<BLANKLINE>
<BLANKLINE>
Overall Statistics :
<BLANKLINE>
95% CI (0.5402,0.99827)
ACC Macro 0.95385
ARI -0.04
AUNP 0.87121
AUNU 0.91212
Bangdiwala B 0.58824
Bennett S 0.74359
CBA 0.75
CSI 0.7
Chi-Squared 91.0
Chi-Squared DF 81
Conditional Entropy 0.46154
Cramer V 0.88192
Cross Entropy 3.39275
F1 Macro 0.81667
F1 Micro 0.76923
FNR Macro 0.15
FNR Micro 0.23077
FPR Macro 0.02576
FPR Micro 0.02564
Gwet AC1 0.7438
Hamming Loss 0.23077
Joint Entropy 3.70044
KL Divergence 0.15385
Kappa 0.74342
Kappa 95% CI (0.48877,0.99807)
Kappa No Prevalence 0.53846
Kappa Standard Error 0.12992
Kappa Unbiased 0.74172
Krippendorff Alpha 0.75166
Lambda A 0.72727
Lambda B 0.72727
Mutual Information 2.77736
NIR 0.15385
Overall ACC 0.76923
Overall CEN 0.09537
Overall J (7.33333,0.73333)
Overall MCC 0.75333
Overall MCEN 0.10746
Overall RACC 0.10059
Overall RACCU 0.10651
P-Value 0.0
PPV Macro 0.85
PPV Micro 0.76923
Pearson C 0.93541
Phi-Squared 7.0
RCI 0.8575
RR 1.3
Reference Entropy 3.2389
Response Entropy 3.2389
SOA1(Landis & Koch) Substantial
SOA2(Fleiss) Intermediate to Good
SOA3(Altman) Good
SOA4(Cicchetti) Excellent
SOA5(Cramer) Very Strong
SOA6(Matthews) Strong
Scott PI 0.74172
Standard Error 0.11685
TNR Macro 0.97424
TNR Micro 0.97436
TPR Macro 0.85
TPR Micro 0.76923
Zero-one Loss 3
<BLANKLINE>
Class Statistics :
<BLANKLINE>
Classes 0 1 2 3 4 5 6 7 8 9
ACC(Accuracy) 1.0 0.92308 0.84615 0.92308 1.0 0.92308 1.0 0.92308 1.0 1.0
AGF(Adjusted F-score) 1.0 0.90468 0.6742 0.71965 1.0 0.71965 1.0 0.90468 1.0 1.0
AGM(Adjusted geometric mean) 1.0 0.93786 0.78186 0.84135 1.0 0.84135 1.0 0.93786 1.0 1.0
AM(Difference between automatic and manual classification) 0 1 0 -1 0 -1 0 1 0 0
AUC(Area under the ROC curve) 1.0 0.95833 0.70455 0.75 1.0 0.75 1.0 0.95833 1.0 1.0
AUCI(AUC value interpretation) Excellent Excellent Good Good Excellent Good Excellent Excellent Excellent Excellent
AUPR(Area under the PR curve) 1.0 0.75 0.5 0.75 1.0 0.75 1.0 0.75 1.0 1.0
BB(Braun-Blanquet similarity) 1.0 0.5 0.5 0.5 1.0 0.5 1.0 0.5 1.0 1.0
BCD(Bray-Curtis dissimilarity) 0.0 0.03846 0.0 0.03846 0.0 0.03846 0.0 0.03846 0.0 0.0
BM(Informedness or bookmaker informedness) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0
CEN(Confusion entropy) 0 0.1267 0.23981 0.1267 0 0.1267 0 0.1267 0 0
DOR(Diagnostic odds ratio) None None 10.0 None None None None None None None
DP(Discriminant power) None None 0.55133 None None None None None None None
DPI(Discriminant power interpretation) None None Poor None None None None None None None
ERR(Error rate) 0.0 0.07692 0.15385 0.07692 0.0 0.07692 0.0 0.07692 0.0 0.0
F0.5(F0.5 score) 1.0 0.55556 0.5 0.83333 1.0 0.83333 1.0 0.55556 1.0 1.0
F1(F1 score - harmonic mean of precision and sensitivity) 1.0 0.66667 0.5 0.66667 1.0 0.66667 1.0 0.66667 1.0 1.0
F2(F2 score) 1.0 0.83333 0.5 0.55556 1.0 0.55556 1.0 0.83333 1.0 1.0
FDR(False discovery rate) 0.0 0.5 0.5 0.0 0.0 0.0 0.0 0.5 0.0 0.0
FN(False negative/miss/type 2 error) 0 0 1 1 0 1 0 0 0 0
FNR(Miss rate or false negative rate) 0.0 0.0 0.5 0.5 0.0 0.5 0.0 0.0 0.0 0.0
FOR(False omission rate) 0.0 0.0 0.09091 0.08333 0.0 0.08333 0.0 0.0 0.0 0.0
FP(False positive/type 1 error/false alarm) 0 1 1 0 0 0 0 1 0 0
FPR(Fall-out or false positive rate) 0.0 0.08333 0.09091 0.0 0.0 0.0 0.0 0.08333 0.0 0.0
G(G-measure geometric mean of precision and sensitivity) 1.0 0.70711 0.5 0.70711 1.0 0.70711 1.0 0.70711 1.0 1.0
GI(Gini index) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0
GM(G-mean geometric mean of specificity and sensitivity) 1.0 0.95743 0.6742 0.70711 1.0 0.70711 1.0 0.95743 1.0 1.0
HD(Hamming distance) 0 1 2 1 0 1 0 1 0 0
IBA(Index of balanced accuracy) 1.0 0.99306 0.2686 0.25 1.0 0.25 1.0 0.99306 1.0 1.0
ICSI(Individual classification success index) 1.0 0.5 0.0 0.5 1.0 0.5 1.0 0.5 1.0 1.0
IS(Information score) 3.70044 2.70044 1.70044 2.70044 3.70044 2.70044 3.70044 2.70044 3.70044 3.70044
J(Jaccard index) 1.0 0.5 0.33333 0.5 1.0 0.5 1.0 0.5 1.0 1.0
LS(Lift score) 13.0 6.5 3.25 6.5 13.0 6.5 13.0 6.5 13.0 13.0
MCC(Matthews correlation coefficient) 1.0 0.677 0.40909 0.677 1.0 0.677 1.0 0.677 1.0 1.0
MCCI(Matthews correlation coefficient interpretation) Very Strong Moderate Weak Moderate Very Strong Moderate Very Strong Moderate Very Strong Very Strong
MCEN(Modified confusion entropy) 0 0.11991 0.2534 0.11991 0 0.11991 0 0.11991 0 0
MK(Markedness) 1.0 0.5 0.40909 0.91667 1.0 0.91667 1.0 0.5 1.0 1.0
N(Condition negative) 12 12 11 11 12 11 12 12 12 12
NLR(Negative likelihood ratio) 0.0 0.0 0.55 0.5 0.0 0.5 0.0 0.0 0.0 0.0
NLRI(Negative likelihood ratio interpretation) Good Good Negligible Negligible Good Negligible Good Good Good Good
NPV(Negative predictive value) 1.0 1.0 0.90909 0.91667 1.0 0.91667 1.0 1.0 1.0 1.0
OC(Overlap coefficient) 1.0 1.0 0.5 1.0 1.0 1.0 1.0 1.0 1.0 1.0
OOC(Otsuka-Ochiai coefficient) 1.0 0.70711 0.5 0.70711 1.0 0.70711 1.0 0.70711 1.0 1.0
OP(Optimized precision) 1.0 0.8796 0.55583 0.58974 1.0 0.58974 1.0 0.8796 1.0 1.0
P(Condition positive or support) 1 1 2 2 1 2 1 1 1 1
PLR(Positive likelihood ratio) None 12.0 5.5 None None None None 12.0 None None
PLRI(Positive likelihood ratio interpretation) None Good Fair None None None None Good None None
POP(Population) 13 13 13 13 13 13 13 13 13 13
PPV(Precision or positive predictive value) 1.0 0.5 0.5 1.0 1.0 1.0 1.0 0.5 1.0 1.0
PRE(Prevalence) 0.07692 0.07692 0.15385 0.15385 0.07692 0.15385 0.07692 0.07692 0.07692 0.07692
Q(Yule Q - coefficient of colligation) None None 0.81818 None None None None None None None
QI(Yule Q interpretation) None None Strong None None None None None None None
RACC(Random accuracy) 0.00592 0.01183 0.02367 0.01183 0.00592 0.01183 0.00592 0.01183 0.00592 0.00592
RACCU(Random accuracy unbiased) 0.00592 0.01331 0.02367 0.01331 0.00592 0.01331 0.00592 0.01331 0.00592 0.00592
TN(True negative/correct rejection) 12 11 10 11 12 11 12 11 12 12
TNR(Specificity or true negative rate) 1.0 0.91667 0.90909 1.0 1.0 1.0 1.0 0.91667 1.0 1.0
TON(Test outcome negative) 12 11 11 12 12 12 12 11 12 12
TOP(Test outcome positive) 1 2 2 1 1 1 1 2 1 1
TP(True positive/hit) 1 1 1 1 1 1 1 1 1 1
TPR(Sensitivity, recall, hit rate, or true positive rate) 1.0 1.0 0.5 0.5 1.0 0.5 1.0 1.0 1.0 1.0
Y(Youden index) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0
dInd(Distance index) 0.0 0.08333 0.5082 0.5 0.0 0.5 0.0 0.08333 0.0 0.0
sInd(Similarity index) 1.0 0.94107 0.64065 0.64645 1.0 0.64645 1.0 0.94107 1.0 1.0
<BLANKLINE>
>>> cm = ConfusionMatrix(matrix={1:{1:22,0:54},0:{1:1,0:57}},transpose=True)
>>> with warns(RuntimeWarning):
... cm.CI("TPR",alpha=2)[1][1][1]
1.0398659919971112
>>> with warns(RuntimeWarning):
... cm.CI("TPR",alpha=2,one_sided=True)[1][1][1]
1.0264713799292524
>>> cm = ConfusionMatrix(matrix={"often":{"often":16,"seldom":6,"never":2},"seldom":{"often":4,"seldom":10,"never":1},"never":{"often":3,"seldom":0,"never":8}})
>>> with warns(RuntimeWarning):
... cm.weighted_kappa()
0.4959042218021425
>>> with warns(RuntimeWarning):
... cm.weighted_kappa(weight={1:{1:1,2:2},2:{1:2,2:1}})
0.4959042218021425
>>> with warns(RuntimeWarning):
... cm.weighted_alpha()
0.5007878978884337
>>> with warns(RuntimeWarning):
... cm.weighted_alpha(weight={1:{1:1,2:2},2:{1:2,2:1}})
0.5007878978884337
>>> y_act=[0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2]
>>> y_pre=[0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,1,1,1,1,1,2,0,1,2,2,2,2]
>>> with warns(RuntimeWarning):
... cm4 = ConfusionMatrix(y_act,y_pre,classes=[1,2,0,3])
>>> cm4
pycm.ConfusionMatrix(classes: [1, 2, 0, 3])
>>> cm4.classes
[1, 2, 0, 3]
>>> cm4.to_array()
array([[5, 1, 3, 0],
[1, 4, 1, 0],
[3, 0, 9, 0],
[0, 0, 0, 0]])
>>> print(cm4)
Predict 1 2 0 3
Actual
1 5 1 3 0
<BLANKLINE>
2 1 4 1 0
<BLANKLINE>
0 3 0 9 0
<BLANKLINE>
3 0 0 0 0
<BLANKLINE>
<BLANKLINE>
<BLANKLINE>
<BLANKLINE>
<BLANKLINE>
Overall Statistics :
<BLANKLINE>
95% CI (0.48885,0.84448)
ACC Macro 0.83333
ARI 0.21053
AUNP None
AUNU None
Bangdiwala B 0.45693
Bennett S 0.55556
CBA None
CSI None
Chi-Squared None
Chi-Squared DF 9
Conditional Entropy 1.08926
Cramer V None
Cross Entropy 1.53762
F1 Macro None
F1 Micro 0.66667
FNR Macro None
FNR Micro 0.33333
FPR Macro 0.13413
FPR Micro 0.11111
Gwet AC1 0.57751
Hamming Loss 0.33333
Joint Entropy 2.61975
KL Divergence None
Kappa 0.47403
Kappa 95% CI (0.19345,0.7546)
Kappa No Prevalence 0.33333
Kappa Standard Error 0.14315
Kappa Unbiased 0.47346
Krippendorff Alpha 0.48321
Lambda A 0.4
Lambda B 0.35714
Mutual Information 0.39731
NIR 0.44444
Overall ACC 0.66667
Overall CEN None
Overall J None
Overall MCC 0.47511
Overall MCEN None
Overall RACC 0.36626
Overall RACCU 0.36694
P-Value 0.01667
PPV Macro None
PPV Micro 0.66667
Pearson C None
Phi-Squared None
RCI 0.2596
RR 6.75
Reference Entropy 1.53049
Response Entropy 1.48657
SOA1(Landis & Koch) Moderate
SOA2(Fleiss) Intermediate to Good
SOA3(Altman) Moderate
SOA4(Cicchetti) Fair
SOA5(Cramer) None
SOA6(Matthews) Weak
Scott PI 0.47346
Standard Error 0.09072
TNR Macro 0.86587
TNR Micro 0.88889
TPR Macro None
TPR Micro 0.66667
Zero-one Loss 9
<BLANKLINE>
Class Statistics :
<BLANKLINE>
Classes 1 2 0 3
ACC(Accuracy) 0.7037 0.88889 0.74074 1.0
AGF(Adjusted F-score) 0.65734 0.79543 0.75595 None
AGM(Adjusted geometric mean) 0.70552 0.86488 0.73866 None
AM(Difference between automatic and manual classification) 0 -1 1 0
AUC(Area under the ROC curve) 0.66667 0.80952 0.74167 None
AUCI(AUC value interpretation) Fair Very Good Good None
AUPR(Area under the PR curve) 0.55556 0.73333 0.72115 None
BB(Braun-Blanquet similarity) 0.55556 0.66667 0.69231 None
BCD(Bray-Curtis dissimilarity) 0.0 0.01852 0.01852 0.0
BM(Informedness or bookmaker informedness) 0.33333 0.61905 0.48333 None
CEN(Confusion entropy) 0.51257 0.36499 0.35586 None
DOR(Diagnostic odds ratio) 4.375 40.0 8.25 None
DP(Discriminant power) 0.35339 0.88326 0.50527 None
DPI(Discriminant power interpretation) Poor Poor Poor None
ERR(Error rate) 0.2963 0.11111 0.25926 0.0
F0.5(F0.5 score) 0.55556 0.76923 0.70312 None
F1(F1 score - harmonic mean of precision and sensitivity) 0.55556 0.72727 0.72 None
F2(F2 score) 0.55556 0.68966 0.7377 None
FDR(False discovery rate) 0.44444 0.2 0.30769 None
FN(False negative/miss/type 2 error) 4 2 3 0
FNR(Miss rate or false negative rate) 0.44444 0.33333 0.25 None
FOR(False omission rate) 0.22222 0.09091 0.21429 0.0
FP(False positive/type 1 error/false alarm) 4 1 4 0
FPR(Fall-out or false positive rate) 0.22222 0.04762 0.26667 0.0
G(G-measure geometric mean of precision and sensitivity) 0.55556 0.7303 0.72058 None
GI(Gini index) 0.33333 0.61905 0.48333 None
GM(G-mean geometric mean of specificity and sensitivity) 0.65734 0.79682 0.74162 None
HD(Hamming distance) 8 3 7 0
IBA(Index of balanced accuracy) 0.33608 0.45351 0.55917 None
ICSI(Individual classification success index) 0.11111 0.46667 0.44231 None
IS(Information score) 0.73697 1.848 0.63941 None
J(Jaccard index) 0.38462 0.57143 0.5625 None
LS(Lift score) 1.66667 3.6 1.55769 None
MCC(Matthews correlation coefficient) 0.33333 0.66254 0.48067 None
MCCI(Matthews correlation coefficient interpretation) Weak Moderate Weak None
MCEN(Modified confusion entropy) 0.59795 0.46544 0.44706 None
MK(Markedness) 0.33333 0.70909 0.47802 None
N(Condition negative) 18 21 15 27
NLR(Negative likelihood ratio) 0.57143 0.35 0.34091 None
NLRI(Negative likelihood ratio interpretation) Negligible Poor Poor None
NPV(Negative predictive value) 0.77778 0.90909 0.78571 1.0
OC(Overlap coefficient) 0.55556 0.8 0.75 None
OOC(Otsuka-Ochiai coefficient) 0.55556 0.7303 0.72058 None
OP(Optimized precision) 0.53704 0.71242 0.7295 None
P(Condition positive or support) 9 6 12 0
PLR(Positive likelihood ratio) 2.5 14.0 2.8125 None
PLRI(Positive likelihood ratio interpretation) Poor Good Poor None
POP(Population) 27 27 27 27
PPV(Precision or positive predictive value) 0.55556 0.8 0.69231 None
PRE(Prevalence) 0.33333 0.22222 0.44444 0.0
Q(Yule Q - coefficient of colligation) 0.62791 0.95122 0.78378 None
QI(Yule Q interpretation) Moderate Strong Strong None
RACC(Random accuracy) 0.11111 0.04115 0.21399 0.0
RACCU(Random accuracy unbiased) 0.11111 0.0415 0.21433 0.0
TN(True negative/correct rejection) 14 20 11 27
TNR(Specificity or true negative rate) 0.77778 0.95238 0.73333 1.0
TON(Test outcome negative) 18 22 14 27
TOP(Test outcome positive) 9 5 13 0
TP(True positive/hit) 5 4 9 0
TPR(Sensitivity, recall, hit rate, or true positive rate) 0.55556 0.66667 0.75 None
Y(Youden index) 0.33333 0.61905 0.48333 None
dInd(Distance index) 0.4969 0.33672 0.36553 None
sInd(Similarity index) 0.64864 0.7619 0.74153 None
<BLANKLINE>
>>> with warns(RuntimeWarning):
... cm4 = ConfusionMatrix([1,1,1,1],[1,1,2,1],classes=[1,"s"])
>>> cm4.to_array()
array([[3, 0],
[0, 0]])
>>> cm4
pycm.ConfusionMatrix(classes: ['1', 's'])
>>> with warns(RuntimeWarning):
... cm4 = ConfusionMatrix([1,1,1,1],[1,1,2,1],classes=(1,2))
>>> cm4.to_array()
array([[3, 1],
[0, 0]])
>>> cm4
pycm.ConfusionMatrix(classes: [1, 2])
>>> with warns(RuntimeWarning):
... crv = PRCurve(actual_vector = [1, 1, 2, 2], probs = [[0.1, 0.9], [0.4, 0.6], [0.35, 0.65], [0.8, 0.2]], classes=[2, 1])
>>> crv
pycm.PRCurve(classes: [2, 1])
"""