Document/Document.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PyCM Document"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Version : 2.1\n",
    "-----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table of contents"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><a href=\"#Overview\">Overview</a></li>\n",
    "    <li><a href=\"#Installation\">Installation</a></li>\n",
    "        <ol>\n",
    "            <li><a href=\"#Source-code\">Source Code</a></li>\n",
    "            <li><a href=\"#PyPI\">PyPI</a></li>\n",
    "            <li><a href=\"#Easy-install\">Easy Install</a></li>\n",
    "        </ol>\n",
    "    &nbsp;\n",
    "    <li><a href=\"#Usage\">Usage</a></li>\n",
    "        <ol>\n",
    "            <li><a href=\"#From-vector\">From Vector</a></li>\n",
    "            <li><a href=\"#Direct-CM\">Direct CM</a></li>\n",
    "            <li><a href=\"#Activation-threshold\">Activation Threshold</a></li>\n",
    "            <li><a href=\"#Load-from-file\">Load From File</a></li>\n",
    "            <li><a href=\"#Sample-weights\">Sample Weights</a></li>\n",
    "            <li><a href=\"#Transpose\">Transpose</a></li>\n",
    "            <li><a href=\"#Relabel\">Relabel</a></li>\n",
    "            <li><a href=\"#Online-help\">Online Help</a></li>\n",
    "            <li><a href=\"#Parameter-recommender\">Parameter Recommender</a></li>\n",
    "            <li><a href=\"#Comapre\">Comapre</a></li>\n",
    "            <li><a href=\"#Acceptable-data-types\">Acceptable Data Types</a></li>\n",
    "        </ol>\n",
    "    &nbsp;\n",
    "    <li><a href=\"#Basic-parameters\">Basic Parameters</a></li>\n",
    "    <ol>\n",
    "        <li><a href=\"#TP-(True-positive)\">True Positive</a></li>\n",
    "        <li><a href=\"#TN-(True-negative)\">True Negative</a></li>\n",
    "        <li><a href=\"#FP-(False-positive)\">False Positive</a></li>\n",
    "        <li><a href=\"#FN-(False-negative)\">False Negative</a></li>\n",
    "        <li><a href=\"#P-(Condition-positive)\">Condition Positive</a></li>\n",
    "        <li><a href=\"#N-(Condition-negative)\">Condition Negative</a></li>\n",
    "        <li><a href=\"#TOP-(Test-outcome-positive)\">Test Outcome Positive</a></li>\n",
    "        <li><a href=\"#TON-(Test-outcome-negative)\">Test Outcome Negative</a></li>\n",
    "        <li><a href=\"#POP-(Population)\">Population</a></li>\n",
    "    </ol>\n",
    "    &nbsp;\n",
    "    <li><a href=\"#Class-statistics\">Class Statistics</a></li>\n",
    "    <ol>\n",
    "        <li><a href=\"#TPR-(True-positive-rate)\">True Positive Rate</a></li>\n",
    "        <li><a href=\"#TNR-(True-negative-rate)\">True Negative Rate</a></li>\n",
    "        <li><a href=\"#PPV-(Positive-predictive-value)\">Positive Predictive Value</a></li>\n",
    "        <li><a href=\"#NPV-(Negative-predictive-value)\">Negative Predictive Value</a></li>\n",
    "        <li><a href=\"#FNR-(False-negative-rate)\">False Negative Rate</a></li>\n",
    "        <li><a href=\"#FPR-(False-positive-rate)\">False Positive Rate</a></li>\n",
    "        <li><a href=\"#FDR-(False-discovery-rate)\">False Discovery Rate</a></li>\n",
    "        <li><a href=\"#FOR-(False-omission-rate)\">False Omission Rate</a></li>\n",
    "        <li><a href=\"#ACC-(Accuracy)\">Accuracy</a></li>\n",
    "        <li><a href=\"#ERR-(Error-rate)\">Error Rate</a></li>\n",
    "        <li><a href=\"#FBeta-Score\">FBeta Score</a></li>\n",
    "        <li><a href=\"#MCC-(Matthews-correlation-coefficient)\">Matthews Correlation Coefficient</a></li>\n",
    "        <li><a href=\"#BM-(Bookmaker-informedness)\">Informedness</a></li>\n",
    "        <li><a href=\"#MK-(Markedness)\">Markedness</a></li>\n",
    "        <li><a href=\"#PLR-(Positive-likelihood-ratio)\">Positive Likelihood Ratio</a></li>\n",
    "        <li><a href=\"#NLR-(Negative-likelihood-ratio)\">Negative Likelihood Ratio</a></li>\n",
    "        <li><a href=\"#DOR-(Diagnostic-odds-ratio)\">Diagnostic Odds Ratio</a></li>\n",
    "        <li><a href=\"#PRE-(Prevalence)\">Prevalence</a></li>\n",
    "        <li><a href=\"#G-(G-measure)\">G-Measure</a></li>\n",
    "        <li><a href=\"#RACC-(Random-accuracy)\">Random Accuracy</a></li>\n",
    "        <li><a href=\"#RACCU-(Random-accuracy-unbiased)\">Random Accuracy Unbiased</a></li>\n",
    "        <li><a href=\"#J-(Jaccard-index)\">Jaccard Index</a></li>\n",
    "        <li><a href=\"#IS-(Information-score)\">Information Score</a></li>\n",
    "        <li><a href=\"#CEN-(Confusion-entropy)\">Confusion Entropy</a></li>\n",
    "        <li><a href=\"#MCEN-(Modified-confusion-entropy)\">Modified Confusion Entropy</a></li>\n",
    "        <li><a href=\"#AUC-(Area-under-the-ROC-curve)\">Area Under The ROC Curve</a></li>\n",
    "        <li><a href=\"#dInd-(Distance-index)\">Distance Index</a></li>\n",
    "        <li><a href=\"#sInd-(Similarity-index)\">Similarity Index</a></li>\n",
    "        <li><a href=\"#DP-(Discriminant-power)\">Discriminant Power</a></li>\n",
    "        <li><a href=\"#Y-(Youden-index)\">Youden Index</a></li>\n",
    "        <li><a href=\"#PLRI-(Positive-likelihood-ratio-interpretation)\">Positive Likelihood Ratio Interpretation</a></li>\n",
    "        <li><a href=\"#DPI-(Discriminant-power-interpretation)\">Discriminant Power Interpretation</a></li>\n",
    "        <li><a href=\"#AUCI-(AUC-value-interpretation)\">AUC Value Interpretation</a></li>\n",
    "        <li><a href=\"#GI-(Gini-index)\">Gini Index</a></li>\n",
    "        <li><a href=\"#LS-(Lift-score)\">Lift Score</a></li>\n",
    "        <li><a href=\"#AM-(Automatic/Manual)\">Automatic/Manual</a></li>\n",
    "        <li><a href=\"#BCD-(Bray-Curtis-dissimilarity)\">Bray-Curtis Dissimilarity</a></li>\n",
    "        <li><a href=\"#OP-(Optimized-precision)\">Optimized Precision</a></li>\n",
    "        <li><a href=\"#IBA-(Index-of-balanced-accuracy)\">Index of Balanced Accuracy</a></li>\n",
    "        <li><a href=\"#GM-(G-mean)\">G-Mean</a></li>\n",
    "        <li><a href=\"#Q-(Yule's-Q)\">Yule's-Q</a></li>\n",
    "        <li><a href=\"#AGM-(Adjusted-G-mean)\">Adjusted G-Mean</a></li> \n",
    "    </ol>\n",
    "    &nbsp;\n",
    "    <li><a href=\"#Overall-statistics\">Overall Statistics</a></li>\n",
    "    <ol>\n",
    "        <li><a href=\"#Kappa\">Kappa</a></li>\n",
    "        <li><a href=\"#Kappa-unbiased\">Kappa Unbiased</a></li>\n",
    "        <li><a href=\"#Kappa-no-prevalence\">Kappa No Prevalence</a></li>\n",
    "        <li><a href=\"#Kappa-standard-error\">Kappa Standard Error</a></li>\n",
    "        <li><a href=\"#Kappa-95%-CI\">Kappa 95% CI</a></li>\n",
    "        <li><a href=\"#Chi-squared\">Chi Squared</a></li>\n",
    "        <li><a href=\"#Chi-squared-DF\">Chi Squared DF</a></li>\n",
    "        <li><a href=\"#Phi-squared\">Phi Squared</a></li>\n",
    "        <li><a href=\"#Cramer's-V\">Cramer's V</a></li>\n",
    "        <li><a href=\"#Standard-error\">Standard Error</a></li>\n",
    "        <li><a href=\"#95%-CI\">95% CI</a></li>\n",
    "        <li><a href=\"#Bennett's-S\">Bennett's S</a></li>\n",
    "        <li><a href=\"#Scott's-Pi\">Scott's PI</a></li>\n",
    "        <li><a href=\"#Gwet's-AC1\">Gwet's AC1</a></li>\n",
    "        <li><a href=\"#Reference-entropy\">Reference Entropy</a></li>\n",
    "        <li><a href=\"#Response-entropy\">Response Entropy</a></li>\n",
    "        <li><a href=\"#Cross-entropy\">Cross Entropy</a></li>\n",
    "        <li><a href=\"#Joint-entropy\">Joint Entropy</a></li>\n",
    "        <li><a href=\"#Conditional-entropy\">Conditional Entropy</a></li>\n",
    "        <li><a href=\"#Kullback-Liebler-divergence\">Kullback-Liebler Divergence</a></li>\n",
    "        <li><a href=\"#Mutual-information\">Mutual Information</a></li>\n",
    "        <li><a href=\"#Goodman-&-Kruskal's-lambda-A\">Goodman-Kruskal's Lambda A</a></li>\n",
    "        <li><a href=\"#Goodman-&-Kruskal's-lambda-B\">Goodman-Kruskal's Lambda B</a></li>\n",
    "        <li><a href=\"#SOA1-(Landis-&-Koch's-benchmark)\">Landis-Koch's Benchmark</a></li>\n",
    "        <li><a href=\"#SOA2-(Fleiss'-benchmark)\">Fleiss' Benchmark</a></li>\n",
    "        <li><a href=\"#SOA3-(Altman's-benchmark)\">Altman's Benchmark</a></li>\n",
    "        <li><a href=\"#SOA4-(Cicchetti's-benchmark)\">Cicchetti's Benchmark</a></li>\n",
    "        <li><a href=\"#Overall_ACC\">Overall Accuracy</a></li>\n",
    "        <li><a href=\"#Overall_RACC\">Overall Random Accuracy</a></li>\n",
    "        <li><a href=\"#Overall_RACCU\">Overall Random Accuracy Unbiased</a></li>\n",
    "        <li><a href=\"#PPV_Micro\">Positive Predictive Value Micro</a></li>\n",
    "        <li><a href=\"#TPR_Micro\">True Positive Rate Micro</a></li>\n",
    "        <li><a href=\"#PPV_Macro\">Positive Predictive Value Macro</a></li>\n",
    "        <li><a href=\"#TPR_Macro\">True Positive Rate Macro</a></li>\n",
    "        <li><a href=\"#Overall_J\">Overall Jaccard Index</a></li>\n",
    "        <li><a href=\"#Hamming-loss\">Hamming Loss</a></li>\n",
    "        <li><a href=\"#Zero-one-loss\">Zero-one Loss</a></li>\n",
    "        <li><a href=\"#NIR-(No-information-rate)\">No Information Rate</a></li>\n",
    "        <li><a href=\"#P-Value\">P Value</a></li>\n",
    "        <li><a href=\"#Overall_CEN\">Overall Confusion Entropy</a></li>\n",
    "        <li><a href=\"#Overall_MCEN\">Overall Modified Confusion Entropy</a></li>\n",
    "        <li><a href=\"#Overall_MCC\">Overall Matthews Correlation Coefficient</a></li>\n",
    "        <li><a href=\"#RR-(Global-performance-index)\">Global Performance Index</a></li>\n",
    "        <li><a href=\"#CBA-(Class-balance-accuracy)\">Class Balance Accuracy</a></li>\n",
    "        <li><a href=\"#AUNU\">AUNU</a></li>\n",
    "        <li><a href=\"#AUNP\">AUNP</a></li>\n",
    "        <li><a href=\"#RCI-(Relative-classifier-information)\">Relative Classifier Information</a></li>\n",
    "        <li><a href=\"#Pearson's-C\">Pearson's C</a></li>\n",
    "    </ol>\n",
    "    &nbsp;\n",
    "    <li><a href=\"#Print\">Print</a></li>\n",
    "    <ol>\n",
    "        <li><a href=\"#Full\">Full</a></li>\n",
    "        <li><a href=\"#Matrix\">Matrix</a></li>\n",
    "        <li><a href=\"#Normalized-matrix\">Normalized Matrix</a></li>\n",
    "        <li><a href=\"#Stat\">Stat</a></li>\n",
    "        <li><a href=\"#Compare-report\">Compare Report</a></li>\n",
    "    </ol>\n",
    "    &nbsp;\n",
    "    <li><a href=\"#Save\">Save</a></li>\n",
    "    <ol>\n",
    "        <li><a href=\"#.pycm-file\">pycm</a></li>\n",
    "        <li><a href=\"#HTML\">HTML</a></li>\n",
    "        <li><a href=\"#CSV\">CSV</a></li>\n",
    "        <li><a href=\"#OBJ\">Object</a></li>\n",
    "        <li><a href=\"#comp\">comp</a></li>\n",
    "    </ol>\n",
    "    &nbsp;\n",
    "    <li><a href=\"#Input-errors\">Input Errors</a></li>\n",
    "    <li><a href=\"#Examples\">Examples</a></li>\n",
    "    <li><a href=\"#References\">References</a></li>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overview"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"text-align:justify;\">\n",
    "PyCM is a multi-class confusion matrix library written in Python that supports both input data vectors and direct matrix, and a proper tool for post-classification model evaluation that supports most classes and overall statistics parameters.\t\n",
    "PyCM is the swiss-army knife of confusion matrices, targeted mainly at data scientists that need a broad array of metrics for predictive models and an accurate evaluation of large variety of classifiers.\n",
    "</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"text-align:center;\">\n",
    "    <img src=\"../Otherfiles/block_diagram.jpg\">\n",
    "</div>\n",
    "<center><p style=\"text-align:center;\">Fig1. ConfusionMatrix Block Diagram</p></center>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Installation\t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Source code\n",
    "- Download [Version 2.1](https://github.com/sepandhaghighi/pycm/archive/v2.1.zip) or [Latest Source ](https://github.com/sepandhaghighi/pycm/archive/dev.zip)\n",
    "- Run `pip install -r requirements.txt` or `pip3 install -r requirements.txt` (Need root access)\n",
    "- Run `python3 setup.py install` or `python setup.py install` (Need root access)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PyPI\n",
    "\n",
    "\n",
    "- Check [Python Packaging User Guide](https://packaging.python.org/installing/)     \n",
    "- Run `pip install pycm==2.1` or `pip3 install pycm==2.1` (Need root access)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Conda\n",
    "\n",
    "- Check [Conda Managing Package](https://conda.io/docs/user-guide/tasks/manage-pkgs.html#installing-packages-from-anaconda-org)\n",
    "- `conda install -c sepandhaghighi pycm` (Need root access)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Easy install\n",
    "\n",
    "- Run `easy_install --upgrade pycm` (Need root access)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Usage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### From vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pycm import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_actu = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]\n",
    "y_pred = [0, 0, 2, 1, 0, 2, 1, 0, 2, 0, 2, 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm = ConfusionMatrix(y_actu, y_pred,digit=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li> <span style=\"color:red;\">Notice </span> :  `digit` (the number of digits to the right of the decimal point in a number) is new in <span style=\"color:red;\">version 0.6</span> (default value : 5)</li>\n",
    "    <li>Only for print and save</li>\n",
    "</ul>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pycm.ConfusionMatrix(classes: [0, 1, 2])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.actual_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 0, 2, 1, 0, 2, 1, 0, 2, 0, 2, 2]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.predict_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ACC': {0: 0.8333333333333334, 1: 0.75, 2: 0.5833333333333334},\n",
       " 'AGM': {0: 0.837285964012303, 1: 0.6919986974962765, 2: 0.6071224016819726},\n",
       " 'AM': {0: 2, 1: -1, 2: -1},\n",
       " 'AUC': {0: 0.8888888888888888, 1: 0.611111111111111, 2: 0.5833333333333333},\n",
       " 'AUCI': {0: 'Very Good', 1: 'Fair', 2: 'Poor'},\n",
       " 'BCD': {0: 0.08333333333333333,\n",
       "  1: 0.041666666666666664,\n",
       "  2: 0.041666666666666664},\n",
       " 'BM': {0: 0.7777777777777777, 1: 0.2222222222222221, 2: 0.16666666666666652},\n",
       " 'CEN': {0: 0.25, 1: 0.49657842846620864, 2: 0.6044162769630221},\n",
       " 'DOR': {0: 'None', 1: 3.999999999999998, 2: 1.9999999999999998},\n",
       " 'DP': {0: 'None', 1: 0.331933069996499, 2: 0.16596653499824957},\n",
       " 'DPI': {0: 'None', 1: 'Poor', 2: 'Poor'},\n",
       " 'ERR': {0: 0.16666666666666663, 1: 0.25, 2: 0.41666666666666663},\n",
       " 'F0.5': {0: 0.6521739130434783,\n",
       "  1: 0.45454545454545453,\n",
       "  2: 0.5769230769230769},\n",
       " 'F1': {0: 0.75, 1: 0.4, 2: 0.5454545454545454},\n",
       " 'F2': {0: 0.8823529411764706, 1: 0.35714285714285715, 2: 0.5172413793103449},\n",
       " 'FDR': {0: 0.4, 1: 0.5, 2: 0.4},\n",
       " 'FN': {0: 0, 1: 2, 2: 3},\n",
       " 'FNR': {0: 0.0, 1: 0.6666666666666667, 2: 0.5},\n",
       " 'FOR': {0: 0.0, 1: 0.19999999999999996, 2: 0.4285714285714286},\n",
       " 'FP': {0: 2, 1: 1, 2: 2},\n",
       " 'FPR': {0: 0.2222222222222222,\n",
       "  1: 0.11111111111111116,\n",
       "  2: 0.33333333333333337},\n",
       " 'G': {0: 0.7745966692414834, 1: 0.408248290463863, 2: 0.5477225575051661},\n",
       " 'GI': {0: 0.7777777777777777, 1: 0.2222222222222221, 2: 0.16666666666666652},\n",
       " 'GM': {0: 0.8819171036881969, 1: 0.5443310539518174, 2: 0.5773502691896257},\n",
       " 'IBA': {0: 0.9506172839506174, 1: 0.1316872427983539, 2: 0.2777777777777778},\n",
       " 'IS': {0: 1.263034405833794, 1: 1.0, 2: 0.2630344058337938},\n",
       " 'J': {0: 0.6, 1: 0.25, 2: 0.375},\n",
       " 'LS': {0: 2.4, 1: 2.0, 2: 1.2},\n",
       " 'MCC': {0: 0.6831300510639732, 1: 0.25819888974716115, 2: 0.1690308509457033},\n",
       " 'MCEN': {0: 0.2643856189774724, 1: 0.5, 2: 0.6875},\n",
       " 'MK': {0: 0.6000000000000001, 1: 0.30000000000000004, 2: 0.17142857142857126},\n",
       " 'N': {0: 9, 1: 9, 2: 6},\n",
       " 'NLR': {0: 0.0, 1: 0.7500000000000001, 2: 0.75},\n",
       " 'NPV': {0: 1.0, 1: 0.8, 2: 0.5714285714285714},\n",
       " 'OP': {0: 0.7083333333333334, 1: 0.2954545454545454, 2: 0.4404761904761905},\n",
       " 'P': {0: 3, 1: 3, 2: 6},\n",
       " 'PLR': {0: 4.5, 1: 2.9999999999999987, 2: 1.4999999999999998},\n",
       " 'PLRI': {0: 'Poor', 1: 'Poor', 2: 'Poor'},\n",
       " 'POP': {0: 12, 1: 12, 2: 12},\n",
       " 'PPV': {0: 0.6, 1: 0.5, 2: 0.6},\n",
       " 'PRE': {0: 0.25, 1: 0.25, 2: 0.5},\n",
       " 'Q': {0: 'None', 1: 0.6, 2: 0.3333333333333333},\n",
       " 'RACC': {0: 0.10416666666666667,\n",
       "  1: 0.041666666666666664,\n",
       "  2: 0.20833333333333334},\n",
       " 'RACCU': {0: 0.1111111111111111,\n",
       "  1: 0.04340277777777778,\n",
       "  2: 0.21006944444444442},\n",
       " 'TN': {0: 7, 1: 8, 2: 4},\n",
       " 'TNR': {0: 0.7777777777777778, 1: 0.8888888888888888, 2: 0.6666666666666666},\n",
       " 'TON': {0: 7, 1: 10, 2: 7},\n",
       " 'TOP': {0: 5, 1: 2, 2: 5},\n",
       " 'TP': {0: 3, 1: 1, 2: 3},\n",
       " 'TPR': {0: 1.0, 1: 0.3333333333333333, 2: 0.5},\n",
       " 'Y': {0: 0.7777777777777777, 1: 0.2222222222222221, 2: 0.16666666666666652},\n",
       " 'dInd': {0: 0.2222222222222222, 1: 0.6758625033664689, 2: 0.6009252125773316},\n",
       " 'sInd': {0: 0.8428651597363228, 1: 0.5220930407198541, 2: 0.5750817072006014}}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.class_stat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `cm.statistic_result` prev versions (0.2 >)</li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'95% CI': (0.30438856248221097, 0.8622781041844558),\n",
       " 'AUNP': 0.6666666666666666,\n",
       " 'AUNU': 0.6944444444444443,\n",
       " 'Bennett S': 0.37500000000000006,\n",
       " 'CBA': 0.4777777777777778,\n",
       " 'Chi-Squared': 6.6,\n",
       " 'Chi-Squared DF': 4,\n",
       " 'Conditional Entropy': 0.9591479170272448,\n",
       " 'Cramer V': 0.5244044240850757,\n",
       " 'Cross Entropy': 1.5935164295556343,\n",
       " 'Gwet AC1': 0.3893129770992367,\n",
       " 'Hamming Loss': 0.41666666666666663,\n",
       " 'Joint Entropy': 2.4591479170272446,\n",
       " 'KL Divergence': 0.09351642955563438,\n",
       " 'Kappa': 0.35483870967741943,\n",
       " 'Kappa 95% CI': (-0.07707577422109269, 0.7867531935759315),\n",
       " 'Kappa No Prevalence': 0.16666666666666674,\n",
       " 'Kappa Standard Error': 0.2203645326012817,\n",
       " 'Kappa Unbiased': 0.34426229508196726,\n",
       " 'Lambda A': 0.16666666666666666,\n",
       " 'Lambda B': 0.42857142857142855,\n",
       " 'Mutual Information': 0.5242078379544426,\n",
       " 'NIR': 0.5,\n",
       " 'Overall ACC': 0.5833333333333334,\n",
       " 'Overall CEN': 0.4638112995385119,\n",
       " 'Overall J': (1.225, 0.4083333333333334),\n",
       " 'Overall MCC': 0.36666666666666664,\n",
       " 'Overall MCEN': 0.5189369467580801,\n",
       " 'Overall RACC': 0.3541666666666667,\n",
       " 'Overall RACCU': 0.3645833333333333,\n",
       " 'P-Value': 0.38720703125,\n",
       " 'PPV Macro': 0.5666666666666668,\n",
       " 'PPV Micro': 0.5833333333333334,\n",
       " 'Pearson C': 0.5956833971812705,\n",
       " 'Phi-Squared': 0.5499999999999999,\n",
       " 'RCI': 0.3494718919696284,\n",
       " 'RR': 4.0,\n",
       " 'Reference Entropy': 1.5,\n",
       " 'Response Entropy': 1.4833557549816874,\n",
       " 'SOA1(Landis & Koch)': 'Fair',\n",
       " 'SOA2(Fleiss)': 'Poor',\n",
       " 'SOA3(Altman)': 'Fair',\n",
       " 'SOA4(Cicchetti)': 'Poor',\n",
       " 'Scott PI': 0.34426229508196726,\n",
       " 'Standard Error': 0.14231876063832777,\n",
       " 'TPR Macro': 0.611111111111111,\n",
       " 'TPR Micro': 0.5833333333333334,\n",
       " 'Zero-one Loss': 5}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.overall_stat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `_` removed from overall statistics names in <span style=\"color:red;\">version 1.6</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {0: 3, 1: 0, 2: 0}, 1: {0: 0, 1: 1, 2: 2}, 2: {0: 2, 1: 1, 2: 3}}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {0: 3, 1: 0, 2: 0}, 1: {0: 0, 1: 1, 2: 2}, 2: {0: 2, 1: 1, 2: 3}}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {0: 1.0, 1: 0.0, 2: 0.0},\n",
       " 1: {0: 0.0, 1: 0.33333, 2: 0.66667},\n",
       " 2: {0: 0.33333, 1: 0.16667, 2: 0.5}}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.normalized_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {0: 1.0, 1: 0.0, 2: 0.0},\n",
       " 1: {0: 0.0, 1: 0.33333, 2: 0.66667},\n",
       " 2: {0: 0.33333, 1: 0.16667, 2: 0.5}}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.normalized_table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> : `matrix`, `normalized_matrix` & `normalized_table` added in <span style=\"color:red;\">version 1.5</span> (changed from print style)</li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_actu = numpy.array([2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2])\n",
    "y_pred = numpy.array([0, 0, 2, 1, 0, 2, 1, 0, 2, 0, 2, 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm = ConfusionMatrix(y_actu, y_pred,digit=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pycm.ConfusionMatrix(classes: [0, 1, 2])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `numpy.array` support in <span style=\"color:red;\">versions > 0.7</span></li>\n",
    "</ul>  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Direct CM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm2 = ConfusionMatrix(matrix={0: {0: 3, 1: 0, 2: 0}, 1: {0: 0, 1: 1, 2: 2}, 2: {0: 2, 1: 1, 2: 3}},digit=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pycm.ConfusionMatrix(classes: [0, 1, 2])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm2.actual_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm2.predict_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm2.classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ACC': {0: 0.8333333333333334, 1: 0.75, 2: 0.5833333333333334},\n",
       " 'AGM': {0: 0.837285964012303, 1: 0.6919986974962765, 2: 0.6071224016819726},\n",
       " 'AM': {0: 2, 1: -1, 2: -1},\n",
       " 'AUC': {0: 0.8888888888888888, 1: 0.611111111111111, 2: 0.5833333333333333},\n",
       " 'AUCI': {0: 'Very Good', 1: 'Fair', 2: 'Poor'},\n",
       " 'BCD': {0: 0.08333333333333333,\n",
       "  1: 0.041666666666666664,\n",
       "  2: 0.041666666666666664},\n",
       " 'BM': {0: 0.7777777777777777, 1: 0.2222222222222221, 2: 0.16666666666666652},\n",
       " 'CEN': {0: 0.25, 1: 0.49657842846620864, 2: 0.6044162769630221},\n",
       " 'DOR': {0: 'None', 1: 3.999999999999998, 2: 1.9999999999999998},\n",
       " 'DP': {0: 'None', 1: 0.331933069996499, 2: 0.16596653499824957},\n",
       " 'DPI': {0: 'None', 1: 'Poor', 2: 'Poor'},\n",
       " 'ERR': {0: 0.16666666666666663, 1: 0.25, 2: 0.41666666666666663},\n",
       " 'F0.5': {0: 0.6521739130434783,\n",
       "  1: 0.45454545454545453,\n",
       "  2: 0.5769230769230769},\n",
       " 'F1': {0: 0.75, 1: 0.4, 2: 0.5454545454545454},\n",
       " 'F2': {0: 0.8823529411764706, 1: 0.35714285714285715, 2: 0.5172413793103449},\n",
       " 'FDR': {0: 0.4, 1: 0.5, 2: 0.4},\n",
       " 'FN': {0: 0, 1: 2, 2: 3},\n",
       " 'FNR': {0: 0.0, 1: 0.6666666666666667, 2: 0.5},\n",
       " 'FOR': {0: 0.0, 1: 0.19999999999999996, 2: 0.4285714285714286},\n",
       " 'FP': {0: 2, 1: 1, 2: 2},\n",
       " 'FPR': {0: 0.2222222222222222,\n",
       "  1: 0.11111111111111116,\n",
       "  2: 0.33333333333333337},\n",
       " 'G': {0: 0.7745966692414834, 1: 0.408248290463863, 2: 0.5477225575051661},\n",
       " 'GI': {0: 0.7777777777777777, 1: 0.2222222222222221, 2: 0.16666666666666652},\n",
       " 'GM': {0: 0.8819171036881969, 1: 0.5443310539518174, 2: 0.5773502691896257},\n",
       " 'IBA': {0: 0.9506172839506174, 1: 0.1316872427983539, 2: 0.2777777777777778},\n",
       " 'IS': {0: 1.263034405833794, 1: 1.0, 2: 0.2630344058337938},\n",
       " 'J': {0: 0.6, 1: 0.25, 2: 0.375},\n",
       " 'LS': {0: 2.4, 1: 2.0, 2: 1.2},\n",
       " 'MCC': {0: 0.6831300510639732, 1: 0.25819888974716115, 2: 0.1690308509457033},\n",
       " 'MCEN': {0: 0.2643856189774724, 1: 0.5, 2: 0.6875},\n",
       " 'MK': {0: 0.6000000000000001, 1: 0.30000000000000004, 2: 0.17142857142857126},\n",
       " 'N': {0: 9, 1: 9, 2: 6},\n",
       " 'NLR': {0: 0.0, 1: 0.7500000000000001, 2: 0.75},\n",
       " 'NPV': {0: 1.0, 1: 0.8, 2: 0.5714285714285714},\n",
       " 'OP': {0: 0.7083333333333334, 1: 0.2954545454545454, 2: 0.4404761904761905},\n",
       " 'P': {0: 3, 1: 3, 2: 6},\n",
       " 'PLR': {0: 4.5, 1: 2.9999999999999987, 2: 1.4999999999999998},\n",
       " 'PLRI': {0: 'Poor', 1: 'Poor', 2: 'Poor'},\n",
       " 'POP': {0: 12, 1: 12, 2: 12},\n",
       " 'PPV': {0: 0.6, 1: 0.5, 2: 0.6},\n",
       " 'PRE': {0: 0.25, 1: 0.25, 2: 0.5},\n",
       " 'Q': {0: 'None', 1: 0.6, 2: 0.3333333333333333},\n",
       " 'RACC': {0: 0.10416666666666667,\n",
       "  1: 0.041666666666666664,\n",
       "  2: 0.20833333333333334},\n",
       " 'RACCU': {0: 0.1111111111111111,\n",
       "  1: 0.04340277777777778,\n",
       "  2: 0.21006944444444442},\n",
       " 'TN': {0: 7, 1: 8, 2: 4},\n",
       " 'TNR': {0: 0.7777777777777778, 1: 0.8888888888888888, 2: 0.6666666666666666},\n",
       " 'TON': {0: 7, 1: 10, 2: 7},\n",
       " 'TOP': {0: 5, 1: 2, 2: 5},\n",
       " 'TP': {0: 3, 1: 1, 2: 3},\n",
       " 'TPR': {0: 1.0, 1: 0.3333333333333333, 2: 0.5},\n",
       " 'Y': {0: 0.7777777777777777, 1: 0.2222222222222221, 2: 0.16666666666666652},\n",
       " 'dInd': {0: 0.2222222222222222, 1: 0.6758625033664689, 2: 0.6009252125773316},\n",
       " 'sInd': {0: 0.8428651597363228, 1: 0.5220930407198541, 2: 0.5750817072006014}}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm2.class_stat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'95% CI': (0.30438856248221097, 0.8622781041844558),\n",
       " 'AUNP': 0.6666666666666666,\n",
       " 'AUNU': 0.6944444444444443,\n",
       " 'Bennett S': 0.37500000000000006,\n",
       " 'CBA': 0.4777777777777778,\n",
       " 'Chi-Squared': 6.6,\n",
       " 'Chi-Squared DF': 4,\n",
       " 'Conditional Entropy': 0.9591479170272448,\n",
       " 'Cramer V': 0.5244044240850757,\n",
       " 'Cross Entropy': 1.5935164295556343,\n",
       " 'Gwet AC1': 0.3893129770992367,\n",
       " 'Hamming Loss': 0.41666666666666663,\n",
       " 'Joint Entropy': 2.4591479170272446,\n",
       " 'KL Divergence': 0.09351642955563438,\n",
       " 'Kappa': 0.35483870967741943,\n",
       " 'Kappa 95% CI': (-0.07707577422109269, 0.7867531935759315),\n",
       " 'Kappa No Prevalence': 0.16666666666666674,\n",
       " 'Kappa Standard Error': 0.2203645326012817,\n",
       " 'Kappa Unbiased': 0.34426229508196726,\n",
       " 'Lambda A': 0.16666666666666666,\n",
       " 'Lambda B': 0.42857142857142855,\n",
       " 'Mutual Information': 0.5242078379544426,\n",
       " 'NIR': 0.5,\n",
       " 'Overall ACC': 0.5833333333333334,\n",
       " 'Overall CEN': 0.4638112995385119,\n",
       " 'Overall J': (1.225, 0.4083333333333334),\n",
       " 'Overall MCC': 0.36666666666666664,\n",
       " 'Overall MCEN': 0.5189369467580801,\n",
       " 'Overall RACC': 0.3541666666666667,\n",
       " 'Overall RACCU': 0.3645833333333333,\n",
       " 'P-Value': 0.38720703125,\n",
       " 'PPV Macro': 0.5666666666666668,\n",
       " 'PPV Micro': 0.5833333333333334,\n",
       " 'Pearson C': 0.5956833971812705,\n",
       " 'Phi-Squared': 0.5499999999999999,\n",
       " 'RCI': 0.3494718919696284,\n",
       " 'RR': 4.0,\n",
       " 'Reference Entropy': 1.5,\n",
       " 'Response Entropy': 1.4833557549816874,\n",
       " 'SOA1(Landis & Koch)': 'Fair',\n",
       " 'SOA2(Fleiss)': 'Poor',\n",
       " 'SOA3(Altman)': 'Fair',\n",
       " 'SOA4(Cicchetti)': 'Poor',\n",
       " 'Scott PI': 0.34426229508196726,\n",
       " 'Standard Error': 0.14231876063832777,\n",
       " 'TPR Macro': 0.611111111111111,\n",
       " 'TPR Micro': 0.5833333333333334,\n",
       " 'Zero-one Loss': 5}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm2.overall_stat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "    <li>In direct matrix mode `actual_vector` and `predict_vector` are empty</li>\n",
    "</ul> "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Activation threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`threshold` is added in `version 0.9` for real value prediction.\t\t\t\n",
    "\t\t\t\t\t\t\n",
    "For more information visit [Example 3](http://www.shaghighi.ir/pycm/doc/Example3.html \"Example3\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.9</span> </li>\n",
    "</ul>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load from file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`file` is added in `version 0.9.5` in order to load saved confusion matrix with `.obj` format generated by `save_obj` method.\n",
    "\n",
    "For more information visit [Example 4](http://www.shaghighi.ir/pycm/doc/Example4.html \"Example4\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.9.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`sample_weight` is added in `version 1.2`\n",
    "\n",
    "For more information visit [Example 5](http://www.shaghighi.ir/pycm/doc/Example5.html \"Example5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.2</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transpose"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`transpose` is added in `version 1.2` in order to transpose input matrix (only in `Direct CM` mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm = ConfusionMatrix(matrix={0: {0: 3, 1: 0, 2: 0}, 1: {0: 0, 1: 1, 2: 2}, 2: {0: 2, 1: 1, 2: 3}},digit=5,transpose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict 0       1       2       \n",
      "Actual\n",
      "0       3       0       2       \n",
      "\n",
      "1       0       1       1       \n",
      "\n",
      "2       0       2       3       \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.print_matrix()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.2</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Relabel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`relabel` method is added in `version 1.5` in order to change ConfusionMatrix classnames."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm.relabel(mapping={0:\"L1\",1:\"L2\",2:\"L3\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pycm.ConfusionMatrix(classes: ['L1', 'L2', 'L3'])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Online help"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`online_help` function is added in `version 1.1` in order to open each statistics definition in web browser"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "\n",
    ">>> from pycm import online_help\n",
    ">>> online_help(\"J\")\n",
    ">>> online_help(\"SOA1(Landis & Koch)\")\n",
    ">>> online_help(2)\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* List of items are available by calling `online_help()` (without argument)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Please choose one parameter : \n",
      "\n",
      "Example : online_help(\"J\") or online_help(2)\n",
      "\n",
      "1-95% CI\n",
      "2-ACC\n",
      "3-AGM\n",
      "4-AM\n",
      "5-AUC\n",
      "6-AUCI\n",
      "7-AUNP\n",
      "8-AUNU\n",
      "9-BCD\n",
      "10-BM\n",
      "11-Bennett S\n",
      "12-CBA\n",
      "13-CEN\n",
      "14-Chi-Squared\n",
      "15-Chi-Squared DF\n",
      "16-Conditional Entropy\n",
      "17-Cramer V\n",
      "18-Cross Entropy\n",
      "19-DOR\n",
      "20-DP\n",
      "21-DPI\n",
      "22-ERR\n",
      "23-F0.5\n",
      "24-F1\n",
      "25-F2\n",
      "26-FDR\n",
      "27-FN\n",
      "28-FNR\n",
      "29-FOR\n",
      "30-FP\n",
      "31-FPR\n",
      "32-G\n",
      "33-GI\n",
      "34-GM\n",
      "35-Gwet AC1\n",
      "36-Hamming Loss\n",
      "37-IBA\n",
      "38-IS\n",
      "39-J\n",
      "40-Joint Entropy\n",
      "41-KL Divergence\n",
      "42-Kappa\n",
      "43-Kappa 95% CI\n",
      "44-Kappa No Prevalence\n",
      "45-Kappa Standard Error\n",
      "46-Kappa Unbiased\n",
      "47-LS\n",
      "48-Lambda A\n",
      "49-Lambda B\n",
      "50-MCC\n",
      "51-MCEN\n",
      "52-MK\n",
      "53-Mutual Information\n",
      "54-N\n",
      "55-NIR\n",
      "56-NLR\n",
      "57-NPV\n",
      "58-OP\n",
      "59-Overall ACC\n",
      "60-Overall CEN\n",
      "61-Overall J\n",
      "62-Overall MCC\n",
      "63-Overall MCEN\n",
      "64-Overall RACC\n",
      "65-Overall RACCU\n",
      "66-P\n",
      "67-P-Value\n",
      "68-PLR\n",
      "69-PLRI\n",
      "70-POP\n",
      "71-PPV\n",
      "72-PPV Macro\n",
      "73-PPV Micro\n",
      "74-PRE\n",
      "75-Pearson C\n",
      "76-Phi-Squared\n",
      "77-Q\n",
      "78-RACC\n",
      "79-RACCU\n",
      "80-RCI\n",
      "81-RR\n",
      "82-Reference Entropy\n",
      "83-Response Entropy\n",
      "84-SOA1(Landis & Koch)\n",
      "85-SOA2(Fleiss)\n",
      "86-SOA3(Altman)\n",
      "87-SOA4(Cicchetti)\n",
      "88-Scott PI\n",
      "89-Standard Error\n",
      "90-TN\n",
      "91-TNR\n",
      "92-TON\n",
      "93-TOP\n",
      "94-TP\n",
      "95-TPR\n",
      "96-TPR Macro\n",
      "97-TPR Micro\n",
      "98-Y\n",
      "99-Zero-one Loss\n",
      "100-dInd\n",
      "101-sInd\n"
     ]
    }
   ],
   "source": [
    "online_help()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parameter recommender"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This option has been added in `version 1.9` in order to recommend most related parameters considering the characteristics of the input dataset. The characteristics according to which the parameters are suggested are balance/imbalance and binary/multiclass. All suggestions can be categorized into three main groups: imbalanced dataset, binary classification for a balanced dataset, and multi-class classification for a balanced dataset. The recommendation lists have been gathered according to the respective paper of each parameter and the capabilities which had been claimed by the paper."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"text-align:center;\">\n",
    "    <img src=\"../Otherfiles/recommendation_block_diagram.jpg\">\n",
    "</div>\n",
    "<center><p style=\"text-align:center;\">Fig2. Parameter Recommender Block Diagram</p></center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.imbalance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.binary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ERR',\n",
       " 'TPR Micro',\n",
       " 'TPR Macro',\n",
       " 'PPV Micro',\n",
       " 'PPV Macro',\n",
       " 'ACC',\n",
       " 'Overall ACC',\n",
       " 'MCC',\n",
       " 'Overall MCC',\n",
       " 'BCD',\n",
       " 'Hamming Loss',\n",
       " 'Zero-one Loss']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.recommended_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  also available in HTML report </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> : The recommender system assumes that the input is the result of classification over the whole data rather than just a part of it. If the confusion matrix is the result of test data classification, the recommendation is not valid. </li>\n",
    "</ul>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comapre"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In `version 2.0` a method for comparing several confusion matrices is introduced. This option is a combination of several overall and class-based benchmarks. Each of the benchmarks evaluates the performance of the classification algorithm from good to poor and give them a numeric score. The score of good performance is 1 and for the poor performance is 0.\n",
    "\n",
    "After that, two scores are calculated for each confusion matrices, overall and class based. The overall score is the average of the score of four overall benchmarks which are Landis & Koch, Fleiss, Altman, and Cicchetti. And with a same manner, the class based score is the average of the score of three class-based benchmarks which are Positive Likelihood Ratio Interpretation, Discriminant Power Interpretation, and AUC value Interpretation. It should be notice that if one of the benchmarks returns none for one of the classes, that benchmarks will be eliminate in total averaging. If user set weights for the classes, the averaging over the value of class-based benchmark scores will transform to a weighted average.\n",
    "\n",
    "If the user set the value of `by_class` boolean input `True`, the best confusion matrix is the one with the maximum class-based score. Otherwise, if a confusion matrix obtain the maximum of the both overall and class-based score, that will be the reported as the best confusion matrix but in any other cases the compare object doesn’t select best confusion matrix."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style=\"text-align:center;\">\n",
    "    <img src=\"../Otherfiles/compare_block_diagram.jpg\">\n",
    "</div>\n",
    "<center><p style=\"text-align:center;\">Fig3. Compare Block Diagram</p></center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm2 = ConfusionMatrix(matrix={0:{0:2,1:50,2:6},1:{0:5,1:50,2:3},2:{0:1,1:7,2:50}})\n",
    "cm3 = ConfusionMatrix(matrix={0:{0:50,1:2,2:6},1:{0:50,1:5,2:3},2:{0:1,1:55,2:2}})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "cp = Compare({\"cm2\":cm2,\"cm3\":cm3})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best : cm2\n",
      "\n",
      "Rank  Name   Class-Score    Overall-Score\n",
      "1     cm2    4.15           1.48333\n",
      "2     cm3    2.75           0.95\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(cp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'cm2': {'class': 4.15, 'overall': 1.48333},\n",
       " 'cm3': {'class': 2.75, 'overall': 0.95}}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cp.scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['cm2', 'cm3']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cp.sorted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pycm.ConfusionMatrix(classes: [0, 1, 2])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cp.best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cm2'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cp.best_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "cp2 = Compare({\"cm2\":cm2,\"cm3\":cm3},by_class=True,weight={0:5,1:1,2:1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best : cm3\n",
      "\n",
      "Rank  Name   Class-Score    Overall-Score\n",
      "1     cm3    8.15           0.95\n",
      "2     cm2    6.95           1.48333\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(cp2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Acceptable data types\t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ConfusionMatrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `actual_vector` : python `list` or numpy `array` of any stringable objects\n",
    "2. `predict_vector` : python `list` or numpy `array` of any stringable objects\n",
    "3. `matrix` : `dict`\n",
    "4. `digit`: `int`\n",
    "5. `threshold` : `FunctionType (function or lambda)`\n",
    "6. `file` : `File object`\n",
    "7. `sample_weight` : python `list` or numpy `array` of numbers\n",
    "8. `transpose` : `bool`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* run `help(ConfusionMatrix)` for more information"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Compare"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `cm_dict` : python `dict` of `ConfusionMatrix` object (`str` : `ConfusionMatrix`)\n",
    "2. `by_class` : `bool`\n",
    "3. `weight` : python `dict` of class weights (`class_name` : `float`)\n",
    "4. `digit`: `int`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* run `help(Compare)` for more information"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basic parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TP (True positive)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A true positive test result is one that detects the condition when the\n",
    "condition is present (correctly identified) [[3]](#References)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 3, 'L2': 1, 'L3': 3}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TN (True negative)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A true negative test result is one that does not detect the condition when\n",
    "the condition is absent correctly rejected) [[3]](#References)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 7, 'L2': 8, 'L3': 4}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FP (False positive)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A false positive test result is one that detects the condition when the\n",
    "condition is absent (incorrectly identified) [[3]](#References)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0, 'L2': 2, 'L3': 3}"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.FP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FN (False negative)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A false negative test result is one that does not detect the condition when\n",
    "the condition is present (incorrectly rejected) [[3]](#References)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 2, 'L2': 1, 'L3': 2}"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.FN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### P (Condition positive)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of positive samples.\n",
    "Also known as support (the number of occurrences of each class in y_true) [[3]](#References)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 5, 'L2': 2, 'L3': 5}"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.P"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### N (Condition negative)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of negative samples [[3]](#References)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 7, 'L2': 10, 'L3': 7}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.N"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TOP (Test outcome positive)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of positive outcomes [[3]](#References)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 3, 'L2': 3, 'L3': 6}"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TOP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TON (Test outcome negative)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of negative outcomes [[3]](#References)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 9, 'L2': 9, 'L3': 6}"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TON"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### POP (Population)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#References)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 12, 'L2': 12, 'L3': 12}"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.POP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* <a href=\"https://en.wikipedia.org/wiki/Confusion_matrix\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Class statistics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TPR (True positive rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sensitivity (also called the true positive rate, the recall, or probability of detection in some fields) measures the proportion of positives that are correctly identified as such (e.g. the percentage of sick people who are correctly identified as having the condition) [[3]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Sensitivity_and_specificity\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$TPR=\\frac{TP}{P}=\\frac{TP}{TP+FN}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.6, 'L2': 0.5, 'L3': 0.6}"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TPR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TNR (True negative rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Specificity (also called the true negative rate) measures the proportion of negatives that are correctly identified as such (e.g. the percentage of healthy people who are correctly identified as not having the condition) [[3]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Sensitivity_and_specificity\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$TNR=\\frac{TN}{N}=\\frac{TN}{TN+FP}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 1.0, 'L2': 0.8, 'L3': 0.5714285714285714}"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TNR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PPV (Positive predictive value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predictive value positive is the proportion of positives that correspond to\n",
    "the presence of the condition [[3]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Positive_and_negative_predictive_values\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$PPV=\\frac{TP}{TP+FP}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 1.0, 'L2': 0.3333333333333333, 'L3': 0.5}"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.PPV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NPV (Negative predictive value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predictive value negative is the proportion of negatives that correspond to\n",
    "the absence of the condition [[3]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Positive_and_negative_predictive_values\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$NPV=\\frac{TN}{TN+FN}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.7777777777777778, 'L2': 0.8888888888888888, 'L3': 0.6666666666666666}"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.NPV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FNR (False negative rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The false negative rate is the proportion of positives which yield negative test outcomes with the test, i.e., the conditional probability of a negative test result given that the condition being looked for is present [[3]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/False_positives_and_false_negatives#False_positive_and_false_negative_rates\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$FNR=\\frac{FN}{P}=\\frac{FN}{FN+TP}=1-TPR$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.4, 'L2': 0.5, 'L3': 0.4}"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.FNR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FPR (False positive rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The false positive rate is the proportion of all negatives that still yield positive test outcomes, i.e., the conditional probability of a positive test result given an event that was not present [[3]](#References).\n",
    "\n",
    "The false positive rate is equal to the significance level. The specificity of the test is equal to 1 minus the false positive rate.\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/False_positives_and_false_negatives#False_positive_and_false_negative_rates\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$FPR=\\frac{FP}{N}=\\frac{FP}{FP+TN}=1-TNR$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.0, 'L2': 0.19999999999999996, 'L3': 0.4285714285714286}"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.FPR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FDR (False discovery rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The false discovery rate (FDR) is a method of conceptualizing the rate of type I errors in null hypothesis testing when conducting multiple comparisons. FDR-controlling procedures are designed to control the expected proportion of \"discoveries\" (rejected null hypotheses) that are false (incorrect rejections) [[3]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/False_discovery_rate\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$FDR=\\frac{FP}{FP+TP}=1-PPV$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.0, 'L2': 0.6666666666666667, 'L3': 0.5}"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.FDR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FOR (False omission rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "False omission rate (FOR) is a statistical method used in multiple hypothesis testing to correct for multiple comparisons and it is the complement of the negative predictive value. It measures the proportion of false negatives which are incorrectly rejected [[3]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Positive_and_negative_predictive_values\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$FOR=\\frac{FN}{FN+TN}=1-NPV$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.2222222222222222,\n",
       " 'L2': 0.11111111111111116,\n",
       " 'L3': 0.33333333333333337}"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.FOR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ACC (Accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The accuracy is the number of correct predictions from all predictions made [[3]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Accuracy_and_precision\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ACC=\\frac{TP+TN}{P+N}=\\frac{TP+TN}{TP+TN+FP+FN}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.8333333333333334, 'L2': 0.75, 'L3': 0.5833333333333334}"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.ACC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ERR (Error rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The accuracy is the number of incorrect predictions from all predictions made [[3]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ERR=\\frac{FP+FN}{P+N}=\\frac{FP+FN}{TP+TN+FP+FN}=1-ACC$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.16666666666666663, 'L2': 0.25, 'L3': 0.41666666666666663}"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.ERR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FBeta-Score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In statistical analysis of classification, the F1 score (also F-score or F-measure) is a measure of a test's accuracy. It considers both the precision p and the recall r of the test to compute the score.\n",
    "The F1 score is the harmonic average of the precision and recall, where F1 score reaches its best value at 1 (perfect precision and recall) and worst at 0 [[3]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/F1_score\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$F_{\\beta}=(1+\\beta^2)\\times \\frac{PPV\\times TPR}{(\\beta^2 \\times PPV)+TPR}=\\frac{(1+\\beta^2) \\times TP}{(1+\\beta^2)\\times TP+FP+\\beta^2 \\times FN}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.75, 'L2': 0.4, 'L3': 0.5454545454545454}"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.F1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.8823529411764706, 'L2': 0.35714285714285715, 'L3': 0.5172413793103449}"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.F05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.6521739130434783, 'L2': 0.45454545454545453, 'L3': 0.5769230769230769}"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.F2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.6144578313253012, 'L2': 0.4857142857142857, 'L3': 0.5930232558139535}"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.F_beta(beta=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MCC (Matthews correlation coefficient)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Matthews correlation coefficient is used in machine learning as a measure of the quality of binary (two-class) classifications, introduced by biochemist Brian W. Matthews in 1975. It takes into account true and false positives and negatives and is generally regarded as a balanced measure which can be used even if the classes are of very different sizes.The MCC is in essence a correlation coefficient between the observed and predicted binary classifications; it returns a value between −1 and +1. A coefficient of +1 represents a perfect prediction, 0 no better than random prediction and −1 indicates total disagreement between prediction and observation [[27]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Matthews_correlation_coefficient\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$MCC=\\frac{TP \\times TN-FP \\times FN}{\\sqrt{(TP+FP)\\times (TP+FN)\\times (TN+FP)\\times (TN+FN)}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.6831300510639732, 'L2': 0.25819888974716115, 'L3': 0.1690308509457033}"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.MCC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BM (Bookmaker informedness)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The informedness of a prediction method as captured by a contingency matrix is defined as the probability that the prediction method will make a correct decision as opposed to guessing and is calculated using the bookmaker algorithm [[2]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$BM=TPR+TNR-1$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.6000000000000001,\n",
       " 'L2': 0.30000000000000004,\n",
       " 'L3': 0.17142857142857126}"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.BM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MK (Markedness)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In statistics and psychology, the social science concept of markedness is quantified as a measure of how much one variable is marked as a predictor or possible cause of another, and is also known as Δp (deltaP) in simple two-choice cases [[2]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$MK=PPV+NPV-1$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.7777777777777777, 'L2': 0.2222222222222221, 'L3': 0.16666666666666652}"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.MK"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PLR (Positive likelihood ratio)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Likelihood ratios are used for assessing the value of performing a diagnostic test. They use the sensitivity and specificity of the test to determine whether a test result usefully changes the probability that a condition (such as a disease state) exists. The first description of the use of likelihood ratios for decision rules was made at a symposium on information theory in 1954 [[28]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Likelihood_ratios_in_diagnostic_testing\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$LR_+=PLR=\\frac{TPR}{FPR}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 'None', 'L2': 2.5000000000000004, 'L3': 1.4}"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.PLR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `LR+` renamed to `PLR` in <span style=\"color:red;\">version 1.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NLR (Negative likelihood ratio)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Likelihood ratios are used for assessing the value of performing a diagnostic test. They use the sensitivity and specificity of the test to determine whether a test result usefully changes the probability that a condition (such as a disease state) exists. The first description of the use of likelihood ratios for decision rules was made at a symposium on information theory in 1954 [[28]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Likelihood_ratios_in_diagnostic_testing\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$LR_-=NLR=\\frac{FNR}{TNR}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.4, 'L2': 0.625, 'L3': 0.7000000000000001}"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.NLR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `LR-` renamed to `NLR` in <span style=\"color:red;\">version 1.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DOR (Diagnostic odds ratio)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The diagnostic odds ratio is a measure of the effectiveness of a diagnostic test. It is defined as the ratio of the odds of the test being positive if the subject has a disease relative to the odds of the test being positive if the subject does not have the disease [[28]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Diagnostic_odds_ratio\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$DOR=\\frac{LR+}{LR-}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 'None', 'L2': 4.000000000000001, 'L3': 1.9999999999999998}"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.DOR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PRE (Prevalence)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prevalence is a statistical concept referring to the number of cases of a disease that are present in a particular population at a given time (Reference Likelihood) [[14]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Prevalence\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Prevalence=\\frac{P}{POP}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.4166666666666667, 'L2': 0.16666666666666666, 'L3': 0.4166666666666667}"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.PRE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### G (G-measure)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Geometric mean of precision and sensitivity, also known as Fowlkes–Mallows index [[3]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Fowlkes%E2%80%93Mallows_index\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$G=\\sqrt{PPV\\times TPR}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.7745966692414834, 'L2': 0.408248290463863, 'L3': 0.5477225575051661}"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.G"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RACC (Random accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The expected accuracy from a strategy of randomly guessing categories according to reference and response distributions [[24]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$RACC=\\frac{TOP \\times P}{POP^2}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.10416666666666667,\n",
       " 'L2': 0.041666666666666664,\n",
       " 'L3': 0.20833333333333334}"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.RACC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RACCU (Random accuracy unbiased)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The expected accuracy from a strategy of randomly guessing categories according to the average of the reference and response distributions [[25]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$RACCU=(\\frac{TOP+P}{2 \\times POP})^2$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.1111111111111111,\n",
       " 'L2': 0.04340277777777778,\n",
       " 'L3': 0.21006944444444442}"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.RACCU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### J (Jaccard index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Jaccard index, also known as Intersection over Union and the Jaccard similarity coefficient (originally coined coefficient de communauté by Paul Jaccard), is a statistic used for comparing the similarity and diversity of sample sets [[29]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Jaccard_index\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$A=Vector_{Actual}$$\n",
    "$$B=Vector_{Predict}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$J(A,B)=\\frac{|A\\cap B|}{|A\\cup B|}=\\frac{|A\\cap B|}{|A|+|B|-|A\\cap B|}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.6, 'L2': 0.25, 'L3': 0.375}"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.J"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.9</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### IS (Information score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The amount of information needed to correctly classify an example into\n",
    "class C, whose prior probability is p(C), is defined as -log2(p(C)) [[18]](#References) [[39]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$IS=-log_2(\\frac{TP+FN}{POP})+log_2(\\frac{TP}{TP+FP})$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 1.2630344058337937, 'L2': 0.9999999999999998, 'L3': 0.26303440583379367}"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.IS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CEN (Confusion entropy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "CEN based upon the concept of entropy for evaluating classifier performances. By exploiting the misclassification information of confusion matrices, the measure evaluates the confusion level of the class distribution of\n",
    "misclassified samples. Both theoretical analysis and statistical results show that the proposed measure is more discriminating than accuracy and RCI while it remains relatively consistent with the two measures. Moreover, it is more capable of measuring how the samples of different classes have been separated from each\n",
    "other. Hence the proposed measure is more precise than the two measures and can substitute for them to evaluate classifiers in classification applications [[17]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_{i,j}^{j}=\\frac{Matrix(i,j)}{\\sum_{k=1}^{|C|}\\Big(Matrix(j,k)+Matrix(k,j)\\Big)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_{i,j}^{i}=\\frac{Matrix(i,j)}{\\sum_{k=1}^{|C|}\\Big(Matrix(i,k)+Matrix(k,i)\\Big)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$CEN_j=-\\sum_{k=1,k\\neq j}^{|C|}\\Bigg(P_{j,k}^jlog_{2(|C|-1)}\\Big(P_{j,k}^j\\Big)+P_{k,j}^jlog_{2(|C|-1)}\\Big(P_{k,j}^j\\Big)\\Bigg)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.25, 'L2': 0.49657842846620864, 'L3': 0.6044162769630221}"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.CEN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MCEN (Modified confusion entropy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Modified version of CEN [[19]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_{i,j}^{j}=\\frac{Matrix(i,j)}{\\sum_{k=1}^{|C|}\\Big(Matrix(j,k)+Matrix(k,j)\\Big)-Matrix(j,j)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_{i,j}^{i}=\\frac{Matrix(i,j)}{\\sum_{k=1}^{|C|}\\Big(Matrix(i,k)+Matrix(k,i)\\Big)-Matrix(i,i)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$MCEN_j=-\\sum_{k=1,k\\neq j}^{|C|}\\Bigg(P_{j,k}^jlog_{2(|C|-1)}\\Big(P_{j,k}^j\\Big)+P_{k,j}^jlog_{2(|C|-1)}\\Big(P_{k,j}^j\\Big)\\Bigg)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.2643856189774724, 'L2': 0.5, 'L3': 0.6875}"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.MCEN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AUC (Area under the ROC curve)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Thus, AUC corresponds to the arithmetic mean of sensitivity and specificity values of each class [[23]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$AUC=\\frac{TNR+TPR}{2}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.8, 'L2': 0.65, 'L3': 0.5857142857142856}"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.AUC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.4</span> </li>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  this is an approximate calculation of AUC </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### dInd (Distance index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Euclidean distance of a ROC point from the top left corner of the ROC space, which can take values between 0 (perfect classification) and sqrt(2) [[23]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$dInd=\\sqrt{(1-TNR)^2+(1-TPR)^2}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.4, 'L2': 0.5385164807134504, 'L3': 0.5862367008195198}"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.dInd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### sInd (Similarity index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "sInd is comprised between 0 (no correct classifications) and 1 (perfect classification) [[23]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$sInd = 1 - \\sqrt{\\frac{(1-TNR)^2+(1-TPR)^2}{2}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.717157287525381, 'L2': 0.6192113447068046, 'L3': 0.5854680534700882}"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.sInd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DP (Discriminant power) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Discriminant power (DP) is a measure that summarizes sensitivity and specificity.\n",
    "The DP has been used mainly in feature selection over imbalanced data [[33]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$X=\\frac{TPR}{1-TPR}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Y=\\frac{TNR}{1-TNR}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$DP=\\frac{\\sqrt{3}}{\\pi}(log_{10}X+log_{10}Y)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 'None', 'L2': 0.33193306999649924, 'L3': 0.1659665349982495}"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.DP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Y (Youden index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Youden’s index, evaluates the algorithm’s ability to avoid failure; it’s derived from sensitivity and\n",
    "specificity and denotes a linear correspondence balanced accuracy.\n",
    "As Youden’s index is a linear transformation of the mean sensitivity and specificity, its values are difficult to\n",
    "interpret, we retain that a higher value of Y indicates better ability to avoid failure. \n",
    "Youden’s index has been conventionally used to evaluate tests diagnostic, improve efficiency of\n",
    "Telemedical prevention [[33]](#References) [[34]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Youden%27s_J_statistic\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\gamma=BM=TPR+TNR-1$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.6000000000000001,\n",
       " 'L2': 0.30000000000000004,\n",
       " 'L3': 0.17142857142857126}"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PLRI (Positive likelihood ratio interpretation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[33]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">PLR</td>\n",
    "           <td style=\"text-align:center\">Model contribution</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 1 ></td>\n",
    "           <td style=\"text-align:center\">Negligible</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">1 - 5</td>\n",
    "           <td style=\"text-align:center\">Poor</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">5 - 10</td>\n",
    "           <td style=\"text-align:center\">Fair</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> > 10 </td>\n",
    "           <td style=\"text-align:center\">Good</td>\n",
    "       </tr>\n",
    "    \n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 'None', 'L2': 'Poor', 'L3': 'Poor'}"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.PLRI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DPI (Discriminant power interpretation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[33]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">DP</td>\n",
    "           <td style=\"text-align:center\">Model contribution</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 1 > </td>\n",
    "           <td style=\"text-align:center\">Poor</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">1 - 2</td>\n",
    "           <td style=\"text-align:center\">Limited</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">2 - 3</td>\n",
    "           <td style=\"text-align:center\">Fair</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> > 3 </td>\n",
    "           <td style=\"text-align:center\">Good</td>\n",
    "       </tr>\n",
    "    \n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 'None', 'L2': 'Poor', 'L3': 'Poor'}"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.DPI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AUCI (AUC value interpretation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[33]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">AUC</td>\n",
    "           <td style=\"text-align:center\">Model performance</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.5 - 0.6</td>\n",
    "           <td style=\"text-align:center\">Poor</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.6 - 0.7</td>\n",
    "           <td style=\"text-align:center\">Fair</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.7 - 0.8</td>\n",
    "           <td style=\"text-align:center\">Good</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 0.8 - 0.9</td>\n",
    "           <td style=\"text-align:center\">Very Good</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.9 - 1.0</td>\n",
    "           <td style=\"text-align:center\">Excellent</td>\n",
    "       </tr>\n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 'Very Good', 'L2': 'Fair', 'L3': 'Poor'}"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.AUCI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.6</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GI (Gini index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A chance-standardized variant of the AUC is given by Gini coefficient, taking values between 0 (no difference\n",
    "between the score distributions of the two classes) and 1 (complete separation between the two distributions).\n",
    "Gini coefficient is widespread use metric in imbalanced data learning [[33]](#References). \n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Gini_coefficient\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$GI=2\\times AUC-1$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.6000000000000001,\n",
       " 'L2': 0.30000000000000004,\n",
       " 'L3': 0.17142857142857126}"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.GI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LS (Lift score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the context of classification, lift compares model predictions to randomly generated predictions. Lift is often used in marketing research combined with gain and lift charts as a visual aid [[35]](#References) [[36]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$LS=\\frac{PPV}{PRE}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 2.4, 'L2': 2.0, 'L3': 1.2}"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.LS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.8</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AM (Automatic/Manual)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Difference between automatic and manual classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$AM=TOP-P=(TP+FP)-(TP+FN)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': -2, 'L2': 1, 'L3': 1}"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.AM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.9</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BCD (Bray-Curtis dissimilarity)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In ecology and biology, the Bray–Curtis dissimilarity, named after J. Roger Bray and John T. Curtis, is a statistic used to quantify the compositional dissimilarity between two different sites, based on counts at each site [[37]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Bray%E2%80%93Curtis_dissimilarity\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$BCD=\\frac{|AM|}{\\sum_{i=1}^{|C|}\\Big(TOP_i+P_i\\Big)}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.08333333333333333,\n",
       " 'L2': 0.041666666666666664,\n",
       " 'L3': 0.041666666666666664}"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.BCD"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.9</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OP (Optimized precision)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Optimized precision is a type of hybrid threshold metrics and has been proposed as a\n",
    "discriminator for building an optimized heuristic classifier. This metric is a combination of\n",
    "accuracy, sensitivity and specificity metrics. The sensitivity and specificity metric were used for \n",
    "stabilizing and optimizing the accuracy performance when dealing with imbalanced class of twoclass problems [[40]](#References) [[42]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$OP = ACC - \\frac{|TNR-TPR|}{|TNR+TPR|}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.5833333333333334, 'L2': 0.5192307692307692, 'L3': 0.5589430894308943}"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.OP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.0</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### IBA (Index of balanced accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The method combines an unbiased index of its overall accuracy and a measure about\n",
    "how dominant is the class with the highest individual accuracy rate [[41]](#References) [[42]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$IBA_{\\alpha}=(1+\\alpha \\times(TPR-TNR))\\times TNR \\times TPR$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.36, 'L2': 0.27999999999999997, 'L3': 0.35265306122448975}"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.IBA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.48, 'L2': 0.34, 'L3': 0.3477551020408163}"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.IBA_alpha(0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.576, 'L2': 0.388, 'L3': 0.34383673469387754}"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.IBA_alpha(0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.0</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GM (G-mean)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Geometric mean of specificity and sensitivity [[3]](#References) [[41]](#References) [[42]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$GM=\\sqrt{TPR \\times TNR}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.7745966692414834, 'L2': 0.6324555320336759, 'L3': 0.5855400437691198}"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.GM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.0</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Q (Yule's Q)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In statistics, Yule's Q, also known as the coefficient of colligation, is a measure of association between two binary variables [[45]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Coefficient_of_colligation\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$OR = \\frac{TP\\times TN}{FP\\times FN}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Q = \\frac{OR-1}{OR+1}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 'None', 'L2': 0.6, 'L3': 0.3333333333333333}"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Q"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AGM (Adjusted G-mean)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "An adjusted version of geometric mean of specificity and sensitivity [[46]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$N_n=\\frac{N}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$AGM=\\frac{GM+TNR\\times N_n}{1+N_n};TPR>0$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$AGM=0;TPR=0$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': 0.8576400016262, 'L2': 0.708612108382005, 'L3': 0.5803410802752335}"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.AGM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overall statistics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kappa"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Kappa is a statistic which measures inter-rater agreement for qualitative (categorical) items. It is generally thought to be a more robust measure than simple percent agreement calculation, as kappa takes into account the possibility of the agreement occurring by chance [[24]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Cohen%27s_kappa\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Kappa=\\frac{ACC_{Overall}-RACC_{Overall}}{1-RACC_{Overall}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.35483870967741943"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Kappa"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kappa unbiased"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The unbiased kappa value is defined in terms of total accuracy and a slightly different computation of expected likelihood that averages the reference and response probabilities [[25]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Kappa_{Unbiased}=\\frac{ACC_{Overall}-RACCU_{Overall}}{1-RACCU_{Overall}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.34426229508196726"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.KappaUnbiased"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kappa no prevalence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The kappa statistic adjusted for prevalence [[14]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Kappa_{NoPrevalence}=2 \\times ACC_{Overall}-1$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.16666666666666674"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.KappaNoPrevalence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kappa standard error"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The standard error(s) of the Kappa coefficient were obtained by Fleiss (1969) [[24]](#References) [[38]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$SE_{Kappa}=\\sqrt{\\frac{ACC_{Overall}\\times (1-RACC_{Overall})}{(1-RACC_{Overall})^2}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.2203645326012817"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Kappa_SE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kappa 95% CI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Kappa 95% Confidence Interval [[24]](#References) [[38]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$CI_{Kappa}=Kappa \\pm 1.96\\times SE_{Kappa}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.07707577422109269, 0.7867531935759315)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Kappa_CI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Chi-squared"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pearson's chi-squared test is a statistical test applied to sets of categorical data to evaluate how likely it is that any observed difference between the sets arose by chance. It is suitable for unpaired data from large samples [[10]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Chi-squared_test\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\chi^2=\\sum_{i=1}^n\\sum_{j=1}^n\\frac{\\Big(Matrix(i,j)-E(i,j)\\Big)^2}{E(i,j)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$E(i,j)=\\frac{TOP_j\\times P_i}{POP}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.6000000000000005"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Chi_Squared"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Chi-squared DF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of degrees of freedom of this confusion matrix for the chi-squared statistic [[10]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$DF=(|C|-1)^2$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.DF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Phi-squared"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In statistics, the phi coefficient (or mean square contingency coefficient) is a measure of association for two binary variables. Introduced by Karl Pearson, this measure is similar to the Pearson correlation coefficient in its interpretation. In fact, a Pearson correlation coefficient estimated for two binary variables will return the phi coefficient [[10]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Phi_coefficient\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\phi^2=\\frac{\\chi^2}{POP}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.55"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Phi_Squared"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cramer's V"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In statistics, Cramér's V (sometimes referred to as Cramér's phi) is a measure of association between two nominal variables, giving a value between 0 and +1 (inclusive). It is based on Pearson's chi-squared statistic and was published by Harald Cramér in 1946 [[26]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Cram%C3%A9r%27s_V\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$V=\\sqrt{\\frac{\\phi^2}{|C|-1}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5244044240850758"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.V"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Standard error"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The standard error (SE) of a statistic (usually an estimate of a parameter) is the standard deviation of its sampling distribution or an estimate of that standard deviation [[31]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Standard_error\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$SE_{ACC}=\\sqrt{\\frac{ACC\\times (1-ACC)}{POP}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.14231876063832777"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.SE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 95% CI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In statistics, a confidence interval (CI) is a type of interval estimate (of a population parameter) that is computed from the observed data. The confidence level is the frequency (i.e., the proportion) of possible confidence intervals that contain the true value of their corresponding parameter. In other words, if confidence intervals are constructed using a given confidence level in an infinite number of independent experiments, the proportion of those intervals that contain the true value of the parameter will match the confidence level [[31]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Confidence_interval\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$CI=ACC \\pm 1.96\\times SE_{ACC}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.30438856248221097, 0.8622781041844558)"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.CI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bennett's S"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Bennett, Alpert & Goldstein’s S is a statistical measure of inter-rater agreement. It was created by Bennett et al. in 1954.\n",
    "Bennett et al. suggested adjusting inter-rater reliability to accommodate the percentage of rater agreement that might be expected by chance was a better measure than simple agreement between raters [[8]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Bennett,_Alpert,_and_Goldstein%E2%80%99s_S\">Wikipedia Page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$p_c=\\frac{1}{|C|}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$S=\\frac{ACC_{Overall}-p_c}{1-p_c}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.37500000000000006"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.S"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scott's Pi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Scott's pi (named after William A. Scott) is a statistic for measuring inter-rater reliability for nominal data in communication studies. Textual entities are annotated with categories by different annotators, and various measures are used to assess the extent of agreement between the annotators, one of which is Scott's pi. Since automatically annotating text is a popular problem in natural language processing, and goal is to get the computer program that is being developed to agree with the humans in the annotations it creates, assessing the extent to which humans agree with each other is important for establishing a reasonable upper limit on computer performance [[7]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Scott%27s_Pi\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$p_c=\\sum_{i=1}^{|C|}(\\frac{TOP_i + P_i}{2\\times POP})^2$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\pi=\\frac{ACC_{Overall}-p_c}{1-p_c}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.34426229508196726"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.PI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gwet's AC1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "AC1 was originally introduced by Gwet in 2001 (Gwet, 2001). The interpretation of AC1 is similar to generalized kappa (Fleiss, 1971), which is used to assess interrater reliability of when there are multiple raters. Gwet (2002) demonstrated that AC1 can overcome the limitations that kappa is sensitive to trait prevalence and rater's classification probabilities (i.e., marginal probabilities), whereas AC1 provides more robust measure of interrater reliability [[6]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\pi_i=\\frac{TOP_i + P_i}{2\\times POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$p_c=\\frac{1}{|C|-1}\\sum_{i=1}^{|C|}\\Big(\\pi_i\\times (1-\\pi_i)\\Big)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$AC_1=\\frac{ACC_{Overall}-p_c}{1-p_c}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3893129770992367"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.AC1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reference entropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The entropy of the decision problem itself as defined by the counts for the reference. The entropy of a distribution is the average negative log probability of outcomes [[30]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Likelihood_{Reference}=\\frac{P_i}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Entropy_{Reference}=-\\sum_{i=1}^{|C|}Likelihood_{Reference}(i)\\times\\log_{2}{Likelihood_{Reference}(i)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$0\\times\\log_{2}{0}\\equiv0$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.4833557549816874"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.ReferenceEntropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Response entropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The entropy of the response distribution. The entropy of a distribution is the average negative log probability of outcomes [[30]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Likelihood_{Response}=\\frac{TOP_i}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Entropy_{Response}=-\\sum_{i=1}^{|C|}Likelihood_{Response}(i)\\times\\log_{2}{Likelihood_{Response}(i)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$0\\times\\log_{2}{0}\\equiv0$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.5"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.ResponseEntropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross entropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The cross-entropy of the response distribution against the reference distribution. The cross-entropy is defined by the negative log probabilities of the response distribution weighted by the reference distribution [[30]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Cross_entropy\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Likelihood_{Reference}=\\frac{P_i}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Likelihood_{Response}=\\frac{TOP_i}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Entropy_{Cross}=-\\sum_{i=1}^{|C|}Likelihood_{Reference}(i)\\times\\log_{2}{Likelihood_{Response}(i)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$0\\times\\log_{2}{0}\\equiv0$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.5833333333333335"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.CrossEntropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Joint entropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The entropy of the joint reference and response distribution as defined by the underlying matrix [[30]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P^{'}(i,j)=\\frac{Matrix(i,j)}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Entropy_{Joint}=-\\sum_{i=1}^{|C|}\\sum_{j=1}^{|C|}P^{'}(i,j)\\times\\log_{2}{P^{'}(i,j)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$0\\times\\log_{2}{0}\\equiv0$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.4591479170272446"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.JointEntropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Conditional entropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The entropy of the distribution of categories in the response given that the reference category was as specified [[30]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Conditional_entropy\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P^{'}(j|i)=\\frac{Matrix(j,i)}{P_i}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Entropy_{Conditional}=\\sum_{i=1}^{|C|}\\Bigg(Likelihood_{Reference}(i)\\times\\Big(-\\sum_{j=1}^{|C|}P^{'}(j|i)\\times\\log_{2}{P^{'}(j|i)}\\Big)\\Bigg)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$0\\times\\log_{2}{0}\\equiv0$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9757921620455572"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.ConditionalEntropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kullback-Liebler divergence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In mathematical statistics, the Kullback–Leibler divergence (also called relative entropy) is a measure of how one probability distribution diverges from a second, expected probability distribution  [[11]](#References) [[30]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence\">Wikipedia Page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Likelihood_{Response}=\\frac{TOP_i}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Likelihood_{Reference}=\\frac{P_i}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Divergence=-\\sum_{i=1}^{|C|}Likelihood_{Reference}\\times\\log_{2}{\\frac{Likelihood_{Reference}}{Likelihood_{Response}}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.09997757835164581"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.KL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mutual information"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Mutual information is defined Kullback-Lieblier divergence, between the product of the individual distributions and the joint distribution.\n",
    "Mutual information is symmetric. We could also subtract the conditional entropy of the reference given the response from the reference entropy to get the same result [[11]](#References) [[30]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Mutual_information\">Wikipedia Page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P^{'}(i,j)=\\frac{Matrix(i,j)}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Likelihood_{Reference}=\\frac{P_i}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Likelihood_{Response}=\\frac{TOP_i}{POP}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$MI=-\\sum_{i=1}^{|C|}\\sum_{j=1}^{|C|}P^{'}(i,j)\\times\\log_{2}\\Big({\\frac{P^{'}(i,j)}{Likelihood_{Reference}(i)\\times Likelihood_{Response}(i) }\\Big)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$MI=Entropy_{Response}-Entropy_{Conditional}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5242078379544428"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.MutualInformation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Goodman & Kruskal's lambda A"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In probability theory and statistics, Goodman & Kruskal's lambda  is a measure of proportional reduction in error in cross tabulation analysis [[12]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Goodman_and_Kruskal%27s_lambda\">Wikipedia page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\lambda_A=\\frac{\\sum_{j=1}^{|C|}Max\\Big(Matrix(-,j)\\Big)-Max(P)}{POP-Max(P)}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.42857142857142855"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.LambdaA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Goodman & Kruskal's lambda B"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In probability theory and statistics, Goodman & Kruskal's lambda  is a measure of proportional reduction in error in cross tabulation analysis [[13]](#References).\n",
    "\n",
    "<a href=\"https://en.wikipedia.org/wiki/Goodman_and_Kruskal%27s_lambda\">Wikipedia Page</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\lambda_B=\\frac{\\sum_{i=1}^{|C|}Max\\Big(Matrix(i,-)\\Big)-Max(TOP)}{POP-Max(TOP)}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.16666666666666666"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.LambdaB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SOA1 (Landis & Koch's benchmark)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[1]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">Kappa</td>\n",
    "           <td style=\"text-align:center\">Strength of Agreement</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 0 ></td>\n",
    "           <td style=\"text-align:center\">Poor</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0 - 0.20</td>\n",
    "           <td style=\"text-align:center\">Slight</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.21 – 0.40</td>\n",
    "           <td style=\"text-align:center\">Fair</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.41 – 0.60</td>\n",
    "           <td style=\"text-align:center\">Moderate</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.61 – 0.80</td>\n",
    "           <td style=\"text-align:center\">Substantial</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.81 – 1.00</td>\n",
    "           <td style=\"text-align:center\">Almost perfect</td>\n",
    "       </tr>\n",
    "    \n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Fair'"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.SOA1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SOA2 (Fleiss' benchmark)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[4]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">Kappa</td>\n",
    "           <td style=\"text-align:center\">Strength of Agreement</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 0.40 ></td>\n",
    "           <td style=\"text-align:center\">Poor</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.4 - 0.75</td>\n",
    "           <td style=\"text-align:center\">Intermediate to Good</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">More than 0.75</td>\n",
    "           <td style=\"text-align:center\">Excellent</td>\n",
    "       </tr>\n",
    "    \n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Poor'"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.SOA2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SOA3 (Altman's benchmark)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[5]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">Kappa</td>\n",
    "           <td style=\"text-align:center\">Strength of Agreement</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 0.2 ></td>\n",
    "           <td style=\"text-align:center\">Poor</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.21 – 0.40</td>\n",
    "           <td style=\"text-align:center\">Fair</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.41 – 0.60</td>\n",
    "           <td style=\"text-align:center\">Moderate</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.61 – 0.80</td>\n",
    "           <td style=\"text-align:center\">Good</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.81 – 1.00</td>\n",
    "           <td style=\"text-align:center\">Very Good</td>\n",
    "       </tr>\n",
    "    \n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Fair'"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.SOA3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SOA4 (Cicchetti's benchmark)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[9]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">Kappa</td>\n",
    "           <td style=\"text-align:center\">Strength of Agreement</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\"> 0.4 ></td>\n",
    "           <td style=\"text-align:center\">Poor</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.4 – 0.59</td>\n",
    "           <td style=\"text-align:center\">Fair</td>\n",
    "       </tr>\n",
    "        <tr>\n",
    "           <td style=\"text-align:center\">0.6 – 0.74</td>\n",
    "           <td style=\"text-align:center\">Good</td>\n",
    "       </tr>\n",
    "       <tr>\n",
    "           <td style=\"text-align:center\">0.74 – 1.00</td>\n",
    "           <td style=\"text-align:center\">Excellent</td>\n",
    "       </tr>\n",
    "    \n",
    "    \n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Poor'"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.SOA4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.7</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overall_ACC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ACC_{Overall}=\\frac{\\sum_{i=1}^{|C|}TP_i}{POP}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5833333333333334"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Overall_ACC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overall_RACC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[24]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$RACC_{Overall}=\\sum_{i=1}^{|C|}RACC_i$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3541666666666667"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Overall_RACC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overall_RACCU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[25]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$RACCU_{Overall}=\\sum_{i=1}^{|C|}RACCU_i$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3645833333333333"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Overall_RACCU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.8.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PPV_Micro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$PPV_{Micro}=\\frac{\\sum_{i=1}^{|C|}TP_i}{\\sum_{i=1}^{|C|}TP_i+FP_i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5833333333333334"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.PPV_Micro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TPR_Micro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$TPR_{Micro}=\\frac{\\sum_{i=1}^{|C|}TP_i}{\\sum_{i=1}^{|C|}TP_i+FN_i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5833333333333334"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TPR_Micro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PPV_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$PPV_{Macro}=\\frac{1}{|C|}\\sum_{i=1}^{|C|}\\frac{TP_i}{TP_i+FP_i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.611111111111111"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.PPV_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TPR_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[3]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$TPR_{Macro}=\\frac{1}{|C|}\\sum_{i=1}^{|C|}\\frac{TP_i}{TP_i+FN_i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5666666666666668"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.TPR_Macro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overall_J"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[29]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$J_{Mean}=\\frac{1}{|C|}\\sum_{i=1}^{|C|}J_i$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$J_{Sum}=\\sum_{i=1}^{|C|}J_i$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$J_{Overall}=(J_{Sum},J_{Mean})$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.225, 0.4083333333333334)"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Overall_J"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.9</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hamming loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The hamming_loss computes the average Hamming loss or Hamming distance between two sets of samples [[31]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$L_{Hamming}=\\frac{1}{POP}\\sum_{i=1}^{|P|}1(y_i \\neq \\widehat{y}_i)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.41666666666666663"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.HammingLoss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.0</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Zero-one loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[31]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$L_{0-1}=\\sum_{i=1}^{|P|}1(y_i \\neq \\widehat{y}_i)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.ZeroOneLoss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.1</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NIR (No information rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The no information error rate is the error rate when the input and output are independent."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$NIR=\\frac{1}{POP}Max(P)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4166666666666667"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.NIR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.2</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### P-Value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[31]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$x=\\sum_{i=1}^{|C|}TP_{i}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$p=NIR$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$n=POP$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P-Value_{(ACC > NIR)}=1-\\sum_{i=1}^{x}\\left(\\begin{array}{c}n\\\\ i\\end{array}\\right)p^{i}(1-p)^{n-i}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.18926430237560654"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.PValue"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.2</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overall_CEN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[17]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_j=\\frac{\\sum_{k=1}^{|C|}\\Big(Matrix(j,k)+Matrix(k,j)\\Big)}{2\\sum_{k,l=1}^{|C|}Matrix(k,l)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$CEN_{Overall}=\\sum_{j=1}^{|C|}P_jCEN_j$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4638112995385119"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Overall_CEN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overall_MCEN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[19]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\alpha=\\begin{cases}1 & |C| > 2\\\\0 & |C| = 2\\end{cases}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$P_j=\\frac{\\sum_{k=1}^{|C|}\\Big(Matrix(j,k)+Matrix(k,j)\\Big)-Matrix(j,j)}{2\\sum_{k,l=1}^{|C|}Matrix(k,l)-\\alpha \\sum_{k=1}^{|C|}Matrix(k,k)}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$MCEN_{Overall}=\\sum_{j=1}^{|C|}P_jMCEN_j$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5189369467580801"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Overall_MCEN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.3</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Overall_MCC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[20]](#References) [[27]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$MCC_{Overall}=\\frac{cov(X,Y)}{\\sqrt{cov(X,X)\\times cov(Y,Y)}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$cov(X,Y)=\\sum_{i,j,k=1}^{|C|}\\Big(Matrix(i,i)Matrix(k,j)-Matrix(j,i)Matrix(i,k)\\Big)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$cov(X,X) = \\sum_{i=1}^{|C|}\\Bigg[\\Big(\\sum_{j=1}^{|C|}Matrix(j,i)\\Big)\\Big(\\sum_{k,l=1,k\\neq i}^{|C|}Matrix(l,k)\\Big)\\Bigg]$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$cov(Y,Y) = \\sum_{i=1}^{|C|}\\Bigg[\\Big(\\sum_{j=1}^{|C|}Matrix(i,j)\\Big)\\Big(\\sum_{k,l=1,k\\neq i}^{|C|}Matrix(k,l)\\Big)\\Bigg]$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.36666666666666664"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.Overall_MCC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RR (Global performance index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[21]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$RR=\\frac{1}{|C|}\\sum_{i,j=1}^{|C|}Matrix(i,j)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.0"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.RR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CBA (Class balance accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For more information visit [[22]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$CBA=\\frac{\\sum_{i=1}^{|C|}\\frac{Matrix(i,i)}{Max(TOP_i,P_i)}}{|C|}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4777777777777778"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.CBA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AUNU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When dealing with multiclass problems, a global measure of classification performances based on the ROC approach (AUNU) has been proposed as the average of single-class measures [[23]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$AUNU=\\frac{\\sum_{i=1}^{|C|}AUC_i}{|C|}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6785714285714285"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.AUNU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AUNP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Another option (AUNP) is that of averaging the AUCi values with weights proportional to the number of samples experimentally belonging to each class, that is, the a priori class distribution [[23]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$AUNP=\\sum_{i=1}^{|C|}\\frac{P_i}{POP}AUC_i$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6857142857142857"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.AUNP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RCI (Relative classifier information)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Performance of different classifiers on the same domain can be measured by\n",
    "comparing relative classifier information while classifier information (mutual information) can be used for comparison across different decision problems [[32]](#References) [[22]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$H_d=-\\sum_{i=1}^{|C|}\\Big(\\frac{\\sum_{l=1}^{|C|}Matrix(i,l)}{\\sum_{h,k=1}^{|C|}Matrix(h,k)}log_2\\frac{\\sum_{l=1}^{|C|}Matrix(i,l)}{\\sum_{h,k=1}^{|C|}Matrix(h,k)}\\Big)=Entropy_{Reference}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$H_o=\\sum_{j=1}^{|C|}\\Big(\\frac{\\sum_{k=1}^{|C|}Matrix(k,j)}{\\sum_{h,l=0}^{|C|}Matrix(h,l)}H_{oj}\\Big)=Entropy_{Conditional}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$H_{oj}=-\\sum_{i=1}^{|C|}\\Big(\\frac{Matrix(i,j)}{\\sum_{k=1}^{|C|}Matrix(k,j)}log_2\\frac{Matrix(i,j)}{\\sum_{k=1}^{|C|}Matrix(k,j)}\\Big)$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$RCI=\\frac{H_d-H_o}{H_d}=\\frac{MI}{Entropy_{Reference}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3533932006492363"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.RCI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 1.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pearson's C"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The contingency coefficient is a coefficient of association that tells whether two variables or data sets are independent or dependent of each other. It is also known as Pearson’s coefficient (not to be confused with Pearson’s coefficient of skewness) [[43]](#References) [[44]](#References)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$C=\\sqrt{\\frac{\\chi^2}{\\chi^2+POP}}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5956833971812706"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.C"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.0</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Print"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Full"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict  L1       L2       L3       \n",
      "Actual\n",
      "L1       3        0        2        \n",
      "\n",
      "L2       0        1        1        \n",
      "\n",
      "L3       0        2        3        \n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Overall Statistics : \n",
      "\n",
      "95% CI                                                            (0.30439,0.86228)\n",
      "AUNP                                                              0.68571\n",
      "AUNU                                                              0.67857\n",
      "Bennett S                                                         0.375\n",
      "CBA                                                               0.47778\n",
      "Chi-Squared                                                       6.6\n",
      "Chi-Squared DF                                                    4\n",
      "Conditional Entropy                                               0.97579\n",
      "Cramer V                                                          0.5244\n",
      "Cross Entropy                                                     1.58333\n",
      "Gwet AC1                                                          0.38931\n",
      "Hamming Loss                                                      0.41667\n",
      "Joint Entropy                                                     2.45915\n",
      "KL Divergence                                                     0.09998\n",
      "Kappa                                                             0.35484\n",
      "Kappa 95% CI                                                      (-0.07708,0.78675)\n",
      "Kappa No Prevalence                                               0.16667\n",
      "Kappa Standard Error                                              0.22036\n",
      "Kappa Unbiased                                                    0.34426\n",
      "Lambda A                                                          0.42857\n",
      "Lambda B                                                          0.16667\n",
      "Mutual Information                                                0.52421\n",
      "NIR                                                               0.41667\n",
      "Overall ACC                                                       0.58333\n",
      "Overall CEN                                                       0.46381\n",
      "Overall J                                                         (1.225,0.40833)\n",
      "Overall MCC                                                       0.36667\n",
      "Overall MCEN                                                      0.51894\n",
      "Overall RACC                                                      0.35417\n",
      "Overall RACCU                                                     0.36458\n",
      "P-Value                                                           0.18926\n",
      "PPV Macro                                                         0.61111\n",
      "PPV Micro                                                         0.58333\n",
      "Pearson C                                                         0.59568\n",
      "Phi-Squared                                                       0.55\n",
      "RCI                                                               0.35339\n",
      "RR                                                                4.0\n",
      "Reference Entropy                                                 1.48336\n",
      "Response Entropy                                                  1.5\n",
      "SOA1(Landis & Koch)                                               Fair\n",
      "SOA2(Fleiss)                                                      Poor\n",
      "SOA3(Altman)                                                      Fair\n",
      "SOA4(Cicchetti)                                                   Poor\n",
      "Scott PI                                                          0.34426\n",
      "Standard Error                                                    0.14232\n",
      "TPR Macro                                                         0.56667\n",
      "TPR Micro                                                         0.58333\n",
      "Zero-one Loss                                                     5\n",
      "\n",
      "Class Statistics :\n",
      "\n",
      "Classes                                                           L1            L2            L3            \n",
      "ACC(Accuracy)                                                     0.83333       0.75          0.58333       \n",
      "AGM(Adjusted geometric mean)                                      0.85764       0.70861       0.58034       \n",
      "AM(Difference between automatic and manual classification)        -2            1             1             \n",
      "AUC(Area under the roc curve)                                     0.8           0.65          0.58571       \n",
      "AUCI(AUC value interpretation)                                    Very Good     Fair          Poor          \n",
      "BCD(Bray-Curtis dissimilarity)                                    0.08333       0.04167       0.04167       \n",
      "BM(Informedness or bookmaker informedness)                        0.6           0.3           0.17143       \n",
      "CEN(Confusion entropy)                                            0.25          0.49658       0.60442       \n",
      "DOR(Diagnostic odds ratio)                                        None          4.0           2.0           \n",
      "DP(Discriminant power)                                            None          0.33193       0.16597       \n",
      "DPI(Discriminant power interpretation)                            None          Poor          Poor          \n",
      "ERR(Error rate)                                                   0.16667       0.25          0.41667       \n",
      "F0.5(F0.5 score)                                                  0.88235       0.35714       0.51724       \n",
      "F1(F1 score - harmonic mean of precision and sensitivity)         0.75          0.4           0.54545       \n",
      "F2(F2 score)                                                      0.65217       0.45455       0.57692       \n",
      "FDR(False discovery rate)                                         0.0           0.66667       0.5           \n",
      "FN(False negative/miss/type 2 error)                              2             1             2             \n",
      "FNR(Miss rate or false negative rate)                             0.4           0.5           0.4           \n",
      "FOR(False omission rate)                                          0.22222       0.11111       0.33333       \n",
      "FP(False positive/type 1 error/false alarm)                       0             2             3             \n",
      "FPR(Fall-out or false positive rate)                              0.0           0.2           0.42857       \n",
      "G(G-measure geometric mean of precision and sensitivity)          0.7746        0.40825       0.54772       \n",
      "GI(Gini index)                                                    0.6           0.3           0.17143       \n",
      "GM(G-mean geometric mean of specificity and sensitivity)          0.7746        0.63246       0.58554       \n",
      "IBA(Index of balanced accuracy)                                   0.36          0.28          0.35265       \n",
      "IS(Information score)                                             1.26303       1.0           0.26303       \n",
      "J(Jaccard index)                                                  0.6           0.25          0.375         \n",
      "LS(Lift score)                                                    2.4           2.0           1.2           \n",
      "MCC(Matthews correlation coefficient)                             0.68313       0.2582        0.16903       \n",
      "MCEN(Modified confusion entropy)                                  0.26439       0.5           0.6875        \n",
      "MK(Markedness)                                                    0.77778       0.22222       0.16667       \n",
      "N(Condition negative)                                             7             10            7             \n",
      "NLR(Negative likelihood ratio)                                    0.4           0.625         0.7           \n",
      "NPV(Negative predictive value)                                    0.77778       0.88889       0.66667       \n",
      "OP(Optimized precision)                                           0.58333       0.51923       0.55894       \n",
      "P(Condition positive or support)                                  5             2             5             \n",
      "PLR(Positive likelihood ratio)                                    None          2.5           1.4           \n",
      "PLRI(Positive likelihood ratio interpretation)                    None          Poor          Poor          \n",
      "POP(Population)                                                   12            12            12            \n",
      "PPV(Precision or positive predictive value)                       1.0           0.33333       0.5           \n",
      "PRE(Prevalence)                                                   0.41667       0.16667       0.41667       \n",
      "Q(Yule Q - coefficient of colligation)                            None          0.6           0.33333       \n",
      "RACC(Random accuracy)                                             0.10417       0.04167       0.20833       \n",
      "RACCU(Random accuracy unbiased)                                   0.11111       0.0434        0.21007       \n",
      "TN(True negative/correct rejection)                               7             8             4             \n",
      "TNR(Specificity or true negative rate)                            1.0           0.8           0.57143       \n",
      "TON(Test outcome negative)                                        9             9             6             \n",
      "TOP(Test outcome positive)                                        3             3             6             \n",
      "TP(True positive/hit)                                             3             1             3             \n",
      "TPR(Sensitivity, recall, hit rate, or true positive rate)         0.6           0.5           0.6           \n",
      "Y(Youden index)                                                   0.6           0.3           0.17143       \n",
      "dInd(Distance index)                                              0.4           0.53852       0.58624       \n",
      "sInd(Similarity index)                                            0.71716       0.61921       0.58547       \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict  L1       L2       L3       \n",
      "Actual\n",
      "L1       3        0        2        \n",
      "\n",
      "L2       0        1        1        \n",
      "\n",
      "L3       0        2        3        \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.print_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': {'L1': 3, 'L2': 0, 'L3': 2},\n",
       " 'L2': {'L1': 0, 'L2': 1, 'L3': 1},\n",
       " 'L3': {'L1': 0, 'L2': 2, 'L3': 3}}"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict  L1       ~        \n",
      "Actual\n",
      "L1       3        2        \n",
      "\n",
      "~        0        7        \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.print_matrix(one_vs_all=True,class_name = \"L1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `one_vs_all` : One-Vs-All mode flag (type : `bool`)\n",
    "2. `class_name` : target class name for One-Vs-All mode (type : `any valid type`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `one_vs_all` option, new in <span style=\"color:red;\">version 1.4 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `matrix()` renamed to `print_matrix()` and `matrix` return confusion matrix as `dict` in <span style=\"color:red;\">version 1.5</span></li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normalized matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict   L1        L2        L3        \n",
      "Actual\n",
      "L1        0.6       0.0       0.4       \n",
      "\n",
      "L2        0.0       0.5       0.5       \n",
      "\n",
      "L3        0.0       0.4       0.6       \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.print_normalized_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'L1': {'L1': 0.6, 'L2': 0.0, 'L3': 0.4},\n",
       " 'L2': {'L1': 0.0, 'L2': 0.5, 'L3': 0.5},\n",
       " 'L3': {'L1': 0.0, 'L2': 0.4, 'L3': 0.6}}"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.normalized_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict   L1        ~         \n",
      "Actual\n",
      "L1        0.6       0.4       \n",
      "\n",
      "~         0.0       1.0       \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.print_normalized_matrix(one_vs_all=True,class_name = \"L1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `one_vs_all` : One-Vs-All mode flag (type : `bool`)\n",
    "2. `class_name` : target class name for One-Vs-All mode (type : `any valid type`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `one_vs_all` option, new in <span style=\"color:red;\">version 1.4 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `normalized_matrix()` renamed to `print_normalized_matrix()` and `normalized_matrix` return normalized confusion matrix as `dict` in <span style=\"color:red;\">version 1.5</span></li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall Statistics : \n",
      "\n",
      "95% CI                                                            (0.30439,0.86228)\n",
      "AUNP                                                              0.68571\n",
      "AUNU                                                              0.67857\n",
      "Bennett S                                                         0.375\n",
      "CBA                                                               0.47778\n",
      "Chi-Squared                                                       6.6\n",
      "Chi-Squared DF                                                    4\n",
      "Conditional Entropy                                               0.97579\n",
      "Cramer V                                                          0.5244\n",
      "Cross Entropy                                                     1.58333\n",
      "Gwet AC1                                                          0.38931\n",
      "Hamming Loss                                                      0.41667\n",
      "Joint Entropy                                                     2.45915\n",
      "KL Divergence                                                     0.09998\n",
      "Kappa                                                             0.35484\n",
      "Kappa 95% CI                                                      (-0.07708,0.78675)\n",
      "Kappa No Prevalence                                               0.16667\n",
      "Kappa Standard Error                                              0.22036\n",
      "Kappa Unbiased                                                    0.34426\n",
      "Lambda A                                                          0.42857\n",
      "Lambda B                                                          0.16667\n",
      "Mutual Information                                                0.52421\n",
      "NIR                                                               0.41667\n",
      "Overall ACC                                                       0.58333\n",
      "Overall CEN                                                       0.46381\n",
      "Overall J                                                         (1.225,0.40833)\n",
      "Overall MCC                                                       0.36667\n",
      "Overall MCEN                                                      0.51894\n",
      "Overall RACC                                                      0.35417\n",
      "Overall RACCU                                                     0.36458\n",
      "P-Value                                                           0.18926\n",
      "PPV Macro                                                         0.61111\n",
      "PPV Micro                                                         0.58333\n",
      "Pearson C                                                         0.59568\n",
      "Phi-Squared                                                       0.55\n",
      "RCI                                                               0.35339\n",
      "RR                                                                4.0\n",
      "Reference Entropy                                                 1.48336\n",
      "Response Entropy                                                  1.5\n",
      "SOA1(Landis & Koch)                                               Fair\n",
      "SOA2(Fleiss)                                                      Poor\n",
      "SOA3(Altman)                                                      Fair\n",
      "SOA4(Cicchetti)                                                   Poor\n",
      "Scott PI                                                          0.34426\n",
      "Standard Error                                                    0.14232\n",
      "TPR Macro                                                         0.56667\n",
      "TPR Micro                                                         0.58333\n",
      "Zero-one Loss                                                     5\n",
      "\n",
      "Class Statistics :\n",
      "\n",
      "Classes                                                           L1            L2            L3            \n",
      "ACC(Accuracy)                                                     0.83333       0.75          0.58333       \n",
      "AGM(Adjusted geometric mean)                                      0.85764       0.70861       0.58034       \n",
      "AM(Difference between automatic and manual classification)        -2            1             1             \n",
      "AUC(Area under the roc curve)                                     0.8           0.65          0.58571       \n",
      "AUCI(AUC value interpretation)                                    Very Good     Fair          Poor          \n",
      "BCD(Bray-Curtis dissimilarity)                                    0.08333       0.04167       0.04167       \n",
      "BM(Informedness or bookmaker informedness)                        0.6           0.3           0.17143       \n",
      "CEN(Confusion entropy)                                            0.25          0.49658       0.60442       \n",
      "DOR(Diagnostic odds ratio)                                        None          4.0           2.0           \n",
      "DP(Discriminant power)                                            None          0.33193       0.16597       \n",
      "DPI(Discriminant power interpretation)                            None          Poor          Poor          \n",
      "ERR(Error rate)                                                   0.16667       0.25          0.41667       \n",
      "F0.5(F0.5 score)                                                  0.88235       0.35714       0.51724       \n",
      "F1(F1 score - harmonic mean of precision and sensitivity)         0.75          0.4           0.54545       \n",
      "F2(F2 score)                                                      0.65217       0.45455       0.57692       \n",
      "FDR(False discovery rate)                                         0.0           0.66667       0.5           \n",
      "FN(False negative/miss/type 2 error)                              2             1             2             \n",
      "FNR(Miss rate or false negative rate)                             0.4           0.5           0.4           \n",
      "FOR(False omission rate)                                          0.22222       0.11111       0.33333       \n",
      "FP(False positive/type 1 error/false alarm)                       0             2             3             \n",
      "FPR(Fall-out or false positive rate)                              0.0           0.2           0.42857       \n",
      "G(G-measure geometric mean of precision and sensitivity)          0.7746        0.40825       0.54772       \n",
      "GI(Gini index)                                                    0.6           0.3           0.17143       \n",
      "GM(G-mean geometric mean of specificity and sensitivity)          0.7746        0.63246       0.58554       \n",
      "IBA(Index of balanced accuracy)                                   0.36          0.28          0.35265       \n",
      "IS(Information score)                                             1.26303       1.0           0.26303       \n",
      "J(Jaccard index)                                                  0.6           0.25          0.375         \n",
      "LS(Lift score)                                                    2.4           2.0           1.2           \n",
      "MCC(Matthews correlation coefficient)                             0.68313       0.2582        0.16903       \n",
      "MCEN(Modified confusion entropy)                                  0.26439       0.5           0.6875        \n",
      "MK(Markedness)                                                    0.77778       0.22222       0.16667       \n",
      "N(Condition negative)                                             7             10            7             \n",
      "NLR(Negative likelihood ratio)                                    0.4           0.625         0.7           \n",
      "NPV(Negative predictive value)                                    0.77778       0.88889       0.66667       \n",
      "OP(Optimized precision)                                           0.58333       0.51923       0.55894       \n",
      "P(Condition positive or support)                                  5             2             5             \n",
      "PLR(Positive likelihood ratio)                                    None          2.5           1.4           \n",
      "PLRI(Positive likelihood ratio interpretation)                    None          Poor          Poor          \n",
      "POP(Population)                                                   12            12            12            \n",
      "PPV(Precision or positive predictive value)                       1.0           0.33333       0.5           \n",
      "PRE(Prevalence)                                                   0.41667       0.16667       0.41667       \n",
      "Q(Yule Q - coefficient of colligation)                            None          0.6           0.33333       \n",
      "RACC(Random accuracy)                                             0.10417       0.04167       0.20833       \n",
      "RACCU(Random accuracy unbiased)                                   0.11111       0.0434        0.21007       \n",
      "TN(True negative/correct rejection)                               7             8             4             \n",
      "TNR(Specificity or true negative rate)                            1.0           0.8           0.57143       \n",
      "TON(Test outcome negative)                                        9             9             6             \n",
      "TOP(Test outcome positive)                                        3             3             6             \n",
      "TP(True positive/hit)                                             3             1             3             \n",
      "TPR(Sensitivity, recall, hit rate, or true positive rate)         0.6           0.5           0.6           \n",
      "Y(Youden index)                                                   0.6           0.3           0.17143       \n",
      "dInd(Distance index)                                              0.4           0.53852       0.58624       \n",
      "sInd(Similarity index)                                            0.71716       0.61921       0.58547       \n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.stat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall Statistics : \n",
      "\n",
      "Kappa                                                             0.35484\n",
      "\n",
      "Class Statistics :\n",
      "\n",
      "Classes                                                           L1            L2            L3            \n",
      "ACC(Accuracy)                                                     0.83333       0.75          0.58333       \n",
      "AUC(Area under the roc curve)                                     0.8           0.65          0.58571       \n",
      "TPR(Sensitivity, recall, hit rate, or true positive rate)         0.6           0.5           0.6           \n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.stat(overall_param=[\"Kappa\"],class_param=[\"ACC\",\"AUC\",\"TPR\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall Statistics : \n",
      "\n",
      "Kappa                                                             0.35484\n",
      "\n",
      "Class Statistics :\n",
      "\n",
      "Classes                                                           L1            L3            \n",
      "ACC(Accuracy)                                                     0.83333       0.58333       \n",
      "AUC(Area under the roc curve)                                     0.8           0.58571       \n",
      "TPR(Sensitivity, recall, hit rate, or true positive rate)         0.6           0.6           \n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.stat(overall_param=[\"Kappa\"],class_param=[\"ACC\",\"AUC\",\"TPR\"],class_name=[\"L1\",\"L3\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `overall_param` : overall statistics names for print (type : `list`)\n",
    "2. `class_param` : class statistics names for print (type : `list`)\n",
    "3. `class_name` : class names for print (sub set of classes) (type : `list`)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `cm.params()` in prev versions (0.2 >) </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `overall_param` & `class_param` , new in <span style=\"color:red;\">version 1.6 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `class_name` , new in <span style=\"color:red;\">version 1.7 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best : cm2\n",
      "\n",
      "Rank  Name   Class-Score    Overall-Score\n",
      "1     cm2    4.15           1.48333\n",
      "2     cm3    2.75           0.95\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cp.print_report()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best : cm2\n",
      "\n",
      "Rank  Name   Class-Score    Overall-Score\n",
      "1     cm2    4.15           1.48333\n",
      "2     cm3    2.75           0.95\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(cp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "if \"Document_Files\" not in os.listdir():\n",
    "    os.mkdir(\"Document_Files\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### .pycm file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_Files\\\\cm1.pycm',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_stat(os.path.join(\"Document_Files\",\"cm1\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_Files\\cm1.pycm\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_Files\\\\cm1_filtered.pycm',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_stat(os.path.join(\"Document_Files\",\"cm1_filtered\"),overall_param=[\"Kappa\"],class_param=[\"ACC\",\"AUC\",\"TPR\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_Files\\cm1_filtered.pycm\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_Files\\\\cm1_filtered2.pycm',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_stat(os.path.join(\"Document_Files\",\"cm1_filtered2\"),overall_param=[\"Kappa\"],class_param=[\"ACC\",\"AUC\",\"TPR\"],class_name=[\"L1\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_Files\\cm1_filtered2.pycm\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': \"[Errno 2] No such file or directory: 'cm1asdasd/.pycm'\",\n",
       " 'Status': False}"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_stat(\"cm1asdasd/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `name` : output file name (type : `str`)\n",
    "2. `address` : flag for address return (type : `bool`)\n",
    "3. `overall_param` : overall statistics names for save (type : `list`)\n",
    "4. `class_param` : class statistics names for save (type : `list`)\n",
    "5. `class_name` : class names for print (sub set of classes) (type : `list`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.4</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `overall_param` & `class_param` , new in <span style=\"color:red;\">version 1.6 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `class_name` , new in <span style=\"color:red;\">version 1.7 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_Files\\\\cm1.html',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_html(os.path.join(\"Document_Files\",\"cm1\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_Files\\cm1.html\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_Files\\\\cm1_filtered.html',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_html(os.path.join(\"Document_Files\",\"cm1_filtered\"),overall_param=[\"Kappa\"],class_param=[\"ACC\",\"AUC\",\"TPR\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_Files\\cm1_filtered.html\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_Files\\\\cm1_filtered2.html',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_html(os.path.join(\"Document_Files\",\"cm1_filtered2\"),overall_param=[\"Kappa\"],class_param=[\"ACC\",\"AUC\",\"TPR\"],class_name=[\"L1\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_Files\\cm1_filtered2.html\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_Files\\\\cm1_colored.html',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_html(os.path.join(\"Document_Files\",\"cm1_colored\"),color=(255, 204, 255))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_Files\\cm1_colored.html\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_Files\\\\cm1_colored2.html',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_html(os.path.join(\"Document_Files\",\"cm1_colored2\"),color=\"Crimson\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_Files\\cm1_colored2.html\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_Files\\\\cm1_normalized.html',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_html(os.path.join(\"Document_Files\",\"cm1_normalized\"),color=\"Crimson\",normalize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_Files\\cm1_normalized.html\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': \"[Errno 2] No such file or directory: 'cm1asdasd/.html'\",\n",
       " 'Status': False}"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_html(\"cm1asdasd/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `name` : output file name (type : `str`)\n",
    "2. `address` : flag for address return (type : `bool`)\n",
    "3. `overall_param` : overall statistics names for save (type : `list`)\n",
    "4. `class_param` : class statistics names for save (type : `list`)\n",
    "5. `class_name` : class names for print (sub set of classes) (type : `list`)\n",
    "6. `color` : matrix color (R,G,B) (type : `tuple`/`str`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `overall_param` & `class_param` , new in <span style=\"color:red;\">version 1.6 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `class_name` , new in <span style=\"color:red;\">version 1.7 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `color`, new in <span style=\"color:red;\">version 1.8 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `color` support <a href=\"https://en.wikipedia.org/wiki/X11_color_names\">X11 color names</a></li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `normalize`, new in <span style=\"color:red;\">version 2.0 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_Files\\\\cm1.csv',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_csv(os.path.join(\"Document_Files\",\"cm1\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_Files\\cm1.csv\">Open Stat File</a>\n",
    "<br/>\n",
    "<a href=\"Document_Files\\cm1_matrix.csv\">Open Matrix File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_Files\\\\cm1_filtered.csv',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_csv(os.path.join(\"Document_Files\",\"cm1_filtered\"),class_param=[\"ACC\",\"AUC\",\"TPR\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_Files\\cm1_filtered.csv\">Open Stat File</a>\n",
    "<br/>\n",
    "<a href=\"Document_Files\\cm1_filtered_matrix.csv\">Open Matrix File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_Files\\\\cm1_filtered2.csv',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_csv(os.path.join(\"Document_Files\",\"cm1_filtered2\"),class_param=[\"ACC\",\"AUC\",\"TPR\"],normalize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_Files\\cm1_filtered2.csv\">Open Stat File</a>\n",
    "<br/>\n",
    "<a href=\"Document_Files\\cm1_filtered2_matrix.csv\">Open Matrix File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_Files\\\\cm1_filtered3.csv',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_csv(os.path.join(\"Document_Files\",\"cm1_filtered3\"),class_param=[\"ACC\",\"AUC\",\"TPR\"],class_name=[\"L1\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_Files\\cm1_filtered3.csv\">Open Stat File</a>\n",
    "<br/>\n",
    "<a href=\"Document_Files\\cm1_filtered3_matrix.csv\">Open Matrix File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': \"[Errno 2] No such file or directory: 'cm1asdasd/.csv'\",\n",
       " 'Status': False}"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_csv(\"cm1asdasd/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `name` : output file name (type : `str`)\n",
    "2. `address` : flag for address return (type : `bool`)\n",
    "4. `class_param` : class statistics names for save (type : `list`)\n",
    "5. `class_name` : class names for print (sub set of classes) (type : `list`)\n",
    "6. `matrix_save` : flag for saving matrix in seperate CSV file (type : `bool`)\n",
    "7. `normalize` : flag for saving normalized matrix instead of matrix (type : `bool`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.6</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `class_param` , new in <span style=\"color:red;\">version 1.6 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `class_name` , new in <span style=\"color:red;\">version 1.7 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  `matrix_save` and `normalize`, new in <span style=\"color:red;\">version 1.9 </span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OBJ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_Files\\\\cm1.obj',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_obj(os.path.join(\"Document_Files\",\"cm1\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_Files\\cm1.obj\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': \"[Errno 2] No such file or directory: 'cm1asdasd/.obj'\",\n",
       " 'Status': False}"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.save_obj(\"cm1asdasd/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `name` : output file name (type : `str`)\n",
    "2. `address` : flag for address return (type : `bool`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 0.9.5</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### comp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_Files\\\\cp.comp',\n",
       " 'Status': True}"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cp.save_report(os.path.join(\"Document_Files\",\"cp\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"Document_Files\\cp.comp\">Open File</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Message': \"[Errno 2] No such file or directory: 'cm1asdasd/.comp'\",\n",
       " 'Status': False}"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cp.save_report(\"cm1asdasd/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. `name` : output file name (type : `str`)\n",
    "2. `address` : flag for address return (type : `bool`)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  new in <span style=\"color:red;\">version 2.0</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Input errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The type of input vectors is assumed to be a list or a NumPy array\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm2=ConfusionMatrix(y_actu, 2)\n",
    "except pycmVectorError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input vectors must have same length\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm3=ConfusionMatrix(y_actu, [1,2,3])\n",
    "except pycmVectorError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input vectors are empty\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm_4 = ConfusionMatrix([], [])\n",
    "except pycmVectorError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input vectors must have same length\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm_5 = ConfusionMatrix([1,1,1,], [1,1,1,1])\n",
    "except pycmVectorError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input confusion matrix format error\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm3=ConfusionMatrix(matrix={})\n",
    "except pycmMatrixError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Type of the input matrix classes is assumed  be the same\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm_4=ConfusionMatrix(matrix={1:{1:2,\"1\":2},\"1\":{1:2,\"1\":3}})\n",
    "except pycmMatrixError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of the classes is lower than 2\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cm_5=ConfusionMatrix(matrix={1:{1:2}})\n",
    "except pycmVectorError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The input type is considered to be dictionary but it's not!\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cp=Compare([cm2,cm3])\n",
    "except pycmCompareError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The domain of all ConfusionMatrix objects must be same! The sample size or the number of classes are different.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cp=Compare({\"cm1\":cm,\"cm2\":cm2})\n",
    "except pycmCompareError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The input is considered to consist of pycm.ConfusionMatrix object but it's not!\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cp=Compare({\"cm1\":[],\"cm2\":cm2})\n",
    "except pycmCompareError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lower than two confusion matrices is given for comparing. The minimum number of confusion matrix for comparing is 2.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cp=Compare({\"cm2\":cm2})\n",
    "except pycmCompareError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The weight type must be dictionary and also must be set for all classes.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    cp=Compare({\"cm1\":cm2,\"cm2\":cm3},by_class=True,weight={1:2,2:0})\n",
    "except pycmCompareError as e:\n",
    "    print(str(e))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<ul>\n",
    "    <li><span style=\"color:red;\">Notice </span> :  updated in <span style=\"color:red;\">version 2.0</span> </li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Examples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Example-1 (Comparison of three different classifiers)\t\n",
    "\n",
    "- [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Example1.ipynb)\n",
    "- [HTML](http://www.pycm.ir/doc/Example1.html)\n",
    "\n",
    "### Example-2 (How to plot via matplotlib)\n",
    "\n",
    "- [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Example2.ipynb)\n",
    "- [HTML](http://www.pycm.ir/doc/Example2.html)\n",
    "\n",
    "### Example-3 (Activation threshold)\n",
    "\n",
    "- [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Example3.ipynb)\n",
    "- [HTML](http://www.pycm.ir/doc/Example3.html)\n",
    "\n",
    "### Example-4 (File)\n",
    "\n",
    "- [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Example4.ipynb)\n",
    "- [HTML](http://www.pycm.ir/doc/Example4.html)\n",
    "\n",
    "### Example-5 (Sample weights)\n",
    "\n",
    "- [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Example5.ipynb)\n",
    "- [HTML](http://www.pycm.ir/doc/Example5.html)\n",
    "\n",
    "### Example-6 (Unbalanced data)\n",
    "\n",
    "- [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Example6.ipynb)\n",
    "- [HTML](http://www.pycm.ir/doc/Example6.html)\n",
    "\n",
    "### Example-7 (How to plot via seaborn+pandas)\n",
    "\n",
    "- [Jupyter Notebook](https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Example7.ipynb)\n",
    "- [HTML](http://www.pycm.ir/doc/Example7.html)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## References"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<blockquote>1- J. R. Landis, G. G. Koch, “The measurement of observer agreement for categorical data. Biometrics,” in International Biometric Society,  pp. 159–174, 1977. </blockquote>\n",
    "\n",
    "<blockquote>2- D. M. W. Powers, “Evaluation: from precision, recall and f-measure to roc, informedness, markedness & correlation,” in Journal of Machine Learning Technologies, pp.37-63, 2011.</blockquote>\n",
    "\n",
    "\n",
    "<blockquote>3-  C. Sammut, G. Webb, “Encyclopedia of Machine Learning” in Springer, 2011.</blockquote>\n",
    "\n",
    "<blockquote>4- J. L. Fleiss, “Measuring nominal scale agreement among many raters,” in Psychological Bulletin, pp. 378-382, 1971. </blockquote>\n",
    "\n",
    "<blockquote>5- D.G. Altman, “Practical Statistics for Medical Research,” in Chapman and Hall, 1990.</blockquote>\n",
    "\n",
    "<blockquote>6- K. L. Gwet, “Computing inter-rater reliability and its variance in the presence of high agreement,” in The British Journal of Mathematical and Statistical Psychology, pp. 29–48, 2008.”</blockquote>\n",
    "\n",
    "<blockquote>7- W. A. Scott, “Reliability of content analysis: The case of nominal scaling,” in Public Opinion Quarterly, pp. 321–325, 1955.</blockquote>\n",
    "\n",
    "<blockquote>8- E. M. Bennett, R. Alpert, and A. C. Goldstein, “Communication through limited response questioning,” in The Public Opinion Quarterly, pp. 303–308, 1954.</blockquote>\n",
    "\n",
    "<blockquote>9- D. V. Cicchetti, \"Guidelines, criteria, and rules of thumb for evaluating normed and standardized assessment instruments in psychology,\" in Psychological Assessment, pp. 284–290, 1994.</blockquote>\n",
    "\n",
    "<blockquote>10- R.B. Davies, \"Algorithm AS155: The Distributions of a Linear Combination of χ2 Random Variables,\" in Journal of the Royal Statistical Society, pp. 323–333, 1980.</blockquote>\n",
    "\n",
    "<blockquote>11- S. Kullback, R. A. Leibler \"On information and sufficiency,\" in Annals of Mathematical Statistics, pp. 79–86, 1951.</blockquote>\n",
    "\n",
    "<blockquote>12- L. A. Goodman, W. H. Kruskal, \"Measures of Association for Cross Classifications, IV: Simplification of Asymptotic Variances,\" in Journal of the American Statistical Association, pp. 415–421, 1972.</blockquote>\n",
    "\n",
    "<blockquote>13- L. A. Goodman, W. H. Kruskal, \"Measures of Association for Cross Classifications III: Approximate Sampling Theory,\" in Journal of the American Statistical Association, pp.  310–364, 1963. </blockquote>\n",
    "\n",
    "<blockquote>14- T. Byrt, J. Bishop and J. B. Carlin, “Bias, prevalence, and kappa,” in Journal of Clinical Epidemiology pp. 423-429, 1993.</blockquote>\n",
    "\n",
    "<blockquote>15- M. Shepperd, D. Bowes, and T. Hall, “Researcher Bias: The Use of Machine Learning in Software Defect Prediction,” in IEEE Transactions on Software Engineering, pp. 603-616, 2014.</blockquote>\n",
    "\n",
    "<blockquote>16- X. Deng, Q. Liu, Y. Deng, and S. Mahadevan, “An improved method to construct basic probability assignment based on the confusion matrix for classification problem, ” in Information Sciences, pp.250-261, 2016.</blockquote>\n",
    "\n",
    "<blockquote>17- J.-M. Wei, X.-J. Yuan, Q.-H. Hu, and S.-Q. J. E. S. w. A. Wang, \"A novel measure for evaluating classifiers,\" in Expert Systems with Applications, pp. 3799-3809, 2010.</blockquote>\n",
    "\n",
    "<blockquote>18- I. Kononenko and I. J. M. L. Bratko, \"Information-based evaluation criterion for classifier's performance,\" in Machine Learning, pp. 67-80, 1991.</blockquote>\n",
    "\n",
    "<blockquote>19- R. Delgado and J. D. Núñez-González, \"Enhancing Confusion Entropy as Measure for Evaluating Classifiers,\" in The 13th International Conference on Soft Computing Models in Industrial and Environmental Applications, pp. 79-89, 2018: Springer.</blockquote>\n",
    "\n",
    "<blockquote>20- J. J. C. b. Gorodkin and chemistry, \"Comparing two K-category assignments by a K-category correlation coefficient,\" in Computational Biology and chemistry, pp. 367-374, 2004.</blockquote>\n",
    "\n",
    "<blockquote>21- C. O. Freitas, J. M. De Carvalho, J. Oliveira, S. B. Aires, and R. Sabourin, \"Confusion matrix disagreement for multiple classifiers,\" in Iberoamerican Congress on Pattern Recognition, pp. 387-396, 2007.</blockquote>\n",
    "\n",
    "<blockquote>22- P. Branco, L. Torgo, and R. P. Ribeiro, \"Relevance-based evaluation metrics for multi-class imbalanced domains,\" in Pacific-Asia Conference on Knowledge Discovery and Data Mining, pp. 698-710, 2017. Springer.</blockquote>\n",
    "\n",
    "<blockquote>23- D. Ballabio, F. Grisoni, R. J. C. Todeschini, and I. L. Systems, \"Multivariate comparison of classification performance measures,\" in Chemometrics and Intelligent Laboratory Systems, pp. 33-44, 2018.</blockquote>\n",
    "\n",
    "<blockquote>24- J. J. E. Cohen and p. measurement, \"A coefficient of agreement for nominal scales,\" in Educational and Psychological Measurement, pp. 37-46, 1960.</blockquote>\n",
    "\n",
    "<blockquote>25- S. Siegel, \"Nonparametric statistics for the behavioral sciences,\" in \tNew York : McGraw-Hill, 1956.</blockquote>\n",
    "\n",
    "<blockquote>26- H. Cramér, \"Mathematical methods of statistics (PMS-9),\"in Princeton university press, 2016.</blockquote>\n",
    "\n",
    "<blockquote>27- B. W. J. B. e. B. A.-P. S. Matthews, \"Comparison of the predicted and observed secondary structure of T4 phage lysozyme,\" in Biochimica et Biophysica Acta (BBA) - Protein Structure, pp. 442-451, 1975.</blockquote>\n",
    "\n",
    "<blockquote>28- J. A. J. S. Swets, \"The relative operating characteristic in psychology: a technique for isolating effects of response bias finds wide use in the study of perception and cognition,\" in American Association for the Advancement of Science, pp. 990-1000, 1973.</blockquote> \n",
    "\n",
    "<blockquote>29- P. J. B. S. V. S. N. Jaccard, \"Étude comparative de la distribution florale dans une portion des Alpes et des Jura,\" in Bulletin de la Société vaudoise des sciences naturelles, pp. 547-579, 1901.</blockquote> \n",
    "\n",
    "<blockquote>30- T. M. Cover and J. A. Thomas, \"Elements of information theory,\" in John Wiley & Sons, 2012.</blockquote> \n",
    "\n",
    "<blockquote>31- E. S. Keeping, \"Introduction to statistical inference,\" in Courier Corporation, 1995.</blockquote>\n",
    "\n",
    "<blockquote>32- V. Sindhwani, P. Bhattacharya, and S. Rakshit, \"Information theoretic feature crediting in multiclass support vector machines,\" in Proceedings of the 2001 SIAM International Conference on Data Mining, pp. 1-18, 2001.</blockquote> \n",
    "\n",
    "<blockquote>33- M. Bekkar, H. K. Djemaa, and T. A. J. J. I. E. A. Alitouche, \"Evaluation measures for models assessment over imbalanced data sets,\" in Journal of Information Engineering and Applications, 2013.</blockquote>\n",
    "\n",
    "<blockquote>34- W. J. J. C. Youden, \"Index for rating diagnostic tests,\" in Cancer, pp. 32-35, 1950.</blockquote>\n",
    "\n",
    "<blockquote>35- S. Brin, R. Motwani, J. D. Ullman, and S. J. A. S. R. Tsur, \"Dynamic itemset counting and implication rules for market basket data,\" in Proceedings of the 1997 ACM SIGMOD international conference on Management of datavol, pp. 255-264, 1997.</blockquote> \n",
    "\n",
    "<blockquote>36- S. J. T. J. o. O. S. S. Raschka, \"MLxtend: Providing machine learning and data science utilities and extensions to Python’s scientific computing stack,\" in Journal of Open Source Software, 2018.</blockquote> \n",
    "\n",
    "<blockquote>37- J. BRAy and J. CuRTIS, \"An ordination of upland forest communities of southern Wisconsin.-ecological Monographs,\" in journal of Ecological Monographs, 1957.</blockquote>\t\n",
    "\n",
    "<blockquote>38- J. L. Fleiss, J. Cohen, and B. S. J. P. B. Everitt, \"Large sample standard errors of kappa and weighted kappa,\" in Psychological Bulletin, p. 323, 1969.</blockquote> \t\n",
    "\n",
    "<blockquote>39- M. Felkin, \"Comparing classification results between n-ary and binary problems,\" in Quality Measures in Data Mining: Springer, pp. 277-301, 2007.</blockquote> \n",
    "\n",
    "<blockquote>40- R. Ranawana and V. Palade, \"Optimized Precision-A new measure for classifier performance evaluation,\" in 2006 IEEE International Conference on Evolutionary Computation, pp. 2254-2261, 2006.</blockquote>\t\n",
    "\n",
    "<blockquote>41- V. García, R. A. Mollineda, and J. S. Sánchez, \"Index of balanced accuracy: A performance measure for skewed class distributions,\" in Iberian Conference on Pattern Recognition and Image Analysis, pp. 441-448, 2009.</blockquote> \n",
    "\n",
    "<blockquote>42- P. Branco, L. Torgo, and R. P. J. A. C. S. Ribeiro, \"A survey of predictive modeling on imbalanced domains,\" in Journal ACM Computing Surveys (CSUR), p. 31, 2016.</blockquote> \n",
    "\n",
    "<blockquote>43- K. Pearson, \"Notes on Regression and Inheritance in the Case of Two Parents,\" in Proceedings of the Royal Society of London, p. 240-242, 1895.</blockquote> \n",
    "\n",
    "<blockquote>44- W. J. I. Conover, New York, \"Practical Nonparametric Statistics,\" in John Wiley and Sons, 1999.</blockquote> \n",
    "\n",
    "<blockquote>45- Yule, G. U, \"On the methods of measuring association between two attributes.\" in Journal of the Royal Statistical Society, pp. 579-652, 1912.</blockquote>\n",
    "\n",
    "<blockquote>46- Batuwita, R. and Palade, V, \"A new performance measure for class imbalance learning. application to bioinformatics problems,\" in Machine Learning and Applications, pp.545–550, 2009.</blockquote>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}