pycm/20190526_pycm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Document\n",
    "- https://github.com/sepandhaghighi/pycm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pycm\n",
      "  Downloading https://files.pythonhosted.org/packages/35/86/14ebc41098fb81eb7e458d234622a9ce50857bb7d6f350b29765273c00f5/pycm-2.1-py2.py3-none-any.whl (45kB)\n",
      "Collecting art>=1.8 (from pycm)\n",
      "  Downloading https://files.pythonhosted.org/packages/6e/cb/12329146ae052d8cc797edec123f87ff54f349af34438d01f1ffe7fc6e1c/art-3.6-py2.py3-none-any.whl (489kB)\n",
      "Requirement already satisfied: numpy>=1.9.0 in /Users/yutaro_ikeda/.pyenv/versions/3.6.5/lib/python3.6/site-packages (from pycm) (1.16.2)\n",
      "Requirement already satisfied: coverage>=4.1 in /Users/yutaro_ikeda/.pyenv/versions/3.6.5/lib/python3.6/site-packages (from art>=1.8->pycm) (4.5.1)\n",
      "Installing collected packages: art, pycm\n",
      "Successfully installed art-3.6 pycm-2.1\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "pip install pycm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ミニマムExample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = [\"cat\", \"ant\", \"cat\", \"cat\", \"ant\", \"bird\"]\n",
    "y_pred = [\"ant\", \"ant\", \"cat\", \"cat\", \"ant\", \"cat\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[2 0 0]\n",
      " [0 0 1]\n",
      " [1 0 2]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "print(confusion_matrix(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict    ant        bird       cat        \n",
      "Actual\n",
      "ant        2          0          0          \n",
      "\n",
      "bird       0          0          1          \n",
      "\n",
      "cat        1          0          2          \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pycm import ConfusionMatrix\n",
    "cm = ConfusionMatrix(actual_vector=y_true, predict_vector=y_pred)\n",
    "cm.print_matrix()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# データ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_wine\n",
    "wine = load_wine()\n",
    "X = wine.data\n",
    "y = wine.target\n",
    "taget_name_map = {idx: name for idx, name in enumerate(wine.target_names)}\n",
    "# => {0: 'class_0', 1: 'class_1', 2: 'class_2'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yutaro_ikeda/.pyenv/versions/3.6.5/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/yutaro_ikeda/.pyenv/versions/3.6.5/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()\n",
    "lr.fit(X, y)\n",
    "y_pred = lr.predict(X)\n",
    "y_true = y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2, 0, 0],\n",
       "       [0, 0, 1],\n",
       "       [1, 0, 2]])"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "confusion_matrix(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# pycm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pycm import ConfusionMatrix\n",
    "cm = ConfusionMatrix(actual_vector=y_true, predict_vector=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict  0        1        2        \n",
      "Actual\n",
      "0        57       2        0        \n",
      "\n",
      "1        0        69       2        \n",
      "\n",
      "2        0        1        47       \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.print_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict       0             1             2             \n",
      "Actual\n",
      "0             0.9661        0.0339        0.0           \n",
      "\n",
      "1             0.0           0.97183       0.02817       \n",
      "\n",
      "2             0.0           0.02083       0.97917       \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.print_normalized_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict   0         ~         \n",
      "Actual\n",
      "0         57        2         \n",
      "\n",
      "~         0         119       \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm.print_matrix(one_vs_all=True, class_name=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {0: 57, 1: 2, 2: 0}, 1: {0: 0, 1: 69, 2: 2}, 2: {0: 0, 1: 1, 2: 47}}"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm.table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ラベルで表示"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = [0, 0, 1, 0, 2, 2, 1, 0, 0, 1, 2, 1]\n",
    "y_pred = [1, 0, 1, 2, 2, 2, 1, 1, 2, 2, 0, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict 0       1       2       \n",
      "Actual\n",
      "0       1       2       2       \n",
      "\n",
      "1       0       3       1       \n",
      "\n",
      "2       1       0       2       \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cm = ConfusionMatrix(actual_vector=y_true, predict_vector=y_pred)\n",
    "cm.print_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict    クラス1       クラス2       クラス3       \n",
      "Actual\n",
      "クラス1       1          2          2          \n",
      "\n",
      "クラス2       0          3          1          \n",
      "\n",
      "クラス3       1          0          2          \n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "mapping = {0: \"クラス1\", 1: \"クラス2\", 2: \"クラス3\"}\n",
    "cm.relabel(mapping=mapping)\n",
    "cm.print_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>クラス1</th>\n",
       "      <th>クラス2</th>\n",
       "      <th>クラス3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>クラス1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>クラス2</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>クラス3</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      クラス1  クラス2  クラス3\n",
       "クラス1     1     0     1\n",
       "クラス2     2     3     0\n",
       "クラス3     2     1     2"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "pd.DataFrame(cm.table)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 比較"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "wine = load_wine()\n",
    "X = wine.data\n",
    "y = wine.target\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yutaro_ikeda/.pyenv/versions/3.6.5/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/yutaro_ikeda/.pyenv/versions/3.6.5/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:469: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Users/yutaro_ikeda/.pyenv/versions/3.6.5/lib/python3.6/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "lr = LogisticRegression()\n",
    "lr.fit(X_train, y_train)\n",
    "y_lr_pred = lr.predict(X_test)\n",
    "\n",
    "svc = SVC()\n",
    "svc.fit(X_train, y_train)\n",
    "y_svc_pred = svc.predict(X_test)\n",
    "\n",
    "y_true = y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm_lr = ConfusionMatrix(actual_vector=y_true, predict_vector=y_lr_pred)\n",
    "cm_svc = ConfusionMatrix(actual_vector=y_true, predict_vector=y_svc_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pycm import Compare\n",
    "cp = Compare({\"cm_lr\": cm_lr,\"cm_svc\": cm_svc})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best : cm_lr\n",
      "\n",
      "Rank  Name      Class-Score   Overall-Score\n",
      "1     cm_lr     3.0           4.0\n",
      "2     cm_svc    0.6           1.11667\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(cp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}