分类器的评价.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "digits=datasets.load_digits()\n",
    "X=digits.data\n",
    "y=digits.target.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 将数据处理成二分类问题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y[digits.target==9]=1\n",
    "y[digits.target!=9]=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train,X_test,y_train,y_test=train_test_split(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1347L, 64L)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(450L, 64L)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "log_reg=LogisticRegression()\n",
    "log_reg.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_predict=log_reg.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.98222222222222222"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_reg.score(X_test,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 计算F1分数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score,recall_score,precision_score,confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion_matrix: [[404   3]\n",
      " [  5  38]]\n",
      "recall_score: 0.883720930233\n",
      "precision_score: 0.926829268293\n",
      "f1_score: 0.904761904762\n"
     ]
    }
   ],
   "source": [
    "print 'confusion_matrix:',confusion_matrix(y_test,y_predict)\n",
    "print 'recall_score:',recall_score(y_test,y_predict)\n",
    "print 'precision_score:',precision_score(y_test,y_predict)\n",
    "print 'f1_score:',f1_score(y_test,y_predict)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 通过调整决策边界均衡精准率和召回率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "decisoin_scores=log_reg.decision_function(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-65.644504448466876"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decisoin_scores.min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13.012238824473318"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decisoin_scores.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_predict_2=np.array(decisoin_scores>5,dtype='int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,\n",
       "       0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_predict_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion_matrix_2: [[407   0]\n",
      " [ 21  22]]\n",
      "recall_score_2: 0.511627906977\n",
      "precision_score_2: 1.0\n",
      "f1_score_2: 0.676923076923\n"
     ]
    }
   ],
   "source": [
    "print 'confusion_matrix_2:',confusion_matrix(y_test,y_predict_2)\n",
    "print 'recall_score_2:',recall_score(y_test,y_predict_2)\n",
    "print 'precision_score_2:',precision_score(y_test,y_predict_2)\n",
    "print 'f1_score_2:',f1_score(y_test,y_predict_2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion_matrix_3: [[383  24]\n",
      " [  2  41]]\n",
      "recall_score_3: 0.953488372093\n",
      "precision_score_3: 0.630769230769\n",
      "f1_score_3: 0.759259259259\n"
     ]
    }
   ],
   "source": [
    "y_predict_3=np.array(decisoin_scores>-5,dtype='int')\n",
    "print 'confusion_matrix_3:',confusion_matrix(y_test,y_predict_3)\n",
    "print 'recall_score_3:',recall_score(y_test,y_predict_3)\n",
    "print 'precision_score_3:',precision_score(y_test,y_predict_3)\n",
    "print 'f1_score_3:',f1_score(y_test,y_predict_3)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 可视化精准率和召回率的变化关系"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "thresholds=np.arange(np.min(decisoin_scores),np.max(decisoin_scores),0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "precisions=[]\n",
    "recalls=[]\n",
    "for i in thresholds:\n",
    "    y_predict=np.array(decisoin_scores>i,dtype='int')\n",
    "    recalls.append(recall_score(y_test,y_predict))\n",
    "    precisions.append(precision_score(y_test,y_predict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8VPW9//HXJ5PJMpMNkhAjARJZ\nFGRVQMRd64JarbQCVm3FrVqwdrm9xau39qe391rtra2t1WpV1LaKS1Ev0mqxWBcUCassBiMESBAI\nIfs6y/f3x5nESQhkcWbOTObzfDzyODNnTuZ8ksA733zP93y/YoxBKaXUwJJgdwFKKaVCT8NdKaUG\nIA13pZQagDTclVJqANJwV0qpAUjDXSmlBiANd6WUGoA03JVSagDScFdKqQEo0a4T5+TkmMLCQrtO\nr5RSMWnt2rUHjTG5PR1nW7gXFhZSXFxs1+mVUiomiciu3hyn3TJKKTUAabgrpdQApOGulFIDkIa7\nUkoNQBruSik1APUY7iLypIgcEJHNR3hdROQhESkVkU0iclLoy1RKKdUXvWm5LwYuOsrrs4DRgY+b\ngUe+fFlKKaW+jB7HuRtj3hGRwqMccjnwjLHW6/tQRLJEJN8Y83mIauxs1wfw2T/D8tZKqX5IGwLT\nbgQRuys5orKDjfx1fQVEybKi543NY9KwrLCeIxQ3MQ0F9gQ9Lw/sOyzcReRmrNY9w4cP79/Zyj+C\ndx7o3+cqpUIsEJYjz4XskfaWchSLV5WxeFVZ1Pz+GZKREhPh3t23q9tfj8aYx4DHAKZOndq/X6Gn\n3W59KKXs9+kK+PPXofFgVId7fYuXoVmpvL/oXLtLiZhQjJYpB4YFPS8A9obgfZVS0c6dbW2bDtpb\nRw8aW724kx12lxFRoQj314BvBUbNzABqw9bfrpSKLq4ca9sY5eHe5sWdbNtUWrbo8asVkeeAs4Ec\nESkH7gacAMaYR4HlwMVAKdAEzA9XsUqpKOMOhPvb90HxEzD6Ajj3roidvqapjURHAk6HkJx45JZ5\nfYuX9BQN906MMVf18LoBFoSsIqVU7HCmWtfAKktg32bY8FzEwt3r83Pm/Supa/ECMCLbxRmjcyjM\ndpOZ6mTGcdkMG+wCrG6Z/MyUiNQVLeLrV5lSKvTOv8favnEnrPmjNdwwAsNSqps81LV4OTYzhW+c\nXMD6PTX8dV0FTW2+jmOcDuHUkTl8eqAh7KNToo2Gu1IqNNw54G2BtkZITgv76aqb2gC44+KxfHXS\nsQAYY9hX18K+2hZWllTS0OLlxWJrpHaKM75mW9FwV0qFRvvF1YMlkFHQ+TWHE1yDQ3q66kYr3Ae7\nkzr2iQj5mankZ6YyZfggAL533ij+tb2SaYWhPX+003BXSoVGer61ffwIY8mvfBpO/FrITrevrgWA\nLJfzqMdluZK4fPLQkJ03Vmi4K6VC47iz4Io/WN0ynRh4/UdwcHtIT/fgP6z3y01PDun7DhQa7kqp\n0HA4YdK87l9bcU/Ix8L7jGFEtosh6fE1Cqa34usKg1LKHu7skN/FWt3o4dwThoT0PQcSbbkrpcLP\nlQP7t8K6ZzrvT0qDcV+DhL61M1u9PhpavQx2JfV8cJzScFdKhV/OGNjwJ3jttsNfu6EAhk3v9tP2\nHGriv5dvo83r79g3JCOZCUOtMeuD0zTcj0TDXSkVfpc9BOfc0XlfZQn8aTbU7zvip71dcoC/bd7H\nCcekk+gQPF7DW58c4LnALOPDA3egqsNpuCulwi/BAZldxr5LoCvmKH3xBxussezLbjudRId1/P66\nFmqaPKQ4EzTcj0LDXSllD1dguuDGqiMeUtXYyiCXsyPYAfIyUsjL0BEyPdHRMkopeyQmQ3IGbPyL\nNeFYNzbsqSE7Tcex94eGu1LKPpPmWePfP3z4sJf21bawuaKOJIfGVH9ot4xSyj4XPwBtTd0uen+w\noRWAa08dEemqBgT9laiUspc7BxorramCgzS2WvO060XT/tFwV0rZy50Dfg+01nXa3eSx5mV3JcXX\n2qehot0ySil7uQNTCNw/stMiH2caQ0mywbk4AYLX/sgeDbe+H5EFQWKZhrtSyl5jLoSz77AW+giy\n/fM6VpZUcs3U4WSkBqb13bsBdqyElhpIHWRDsbFDw10pZa/ULDh70WG7V7+/k/u3bGXeuedD+4Ic\nG5dY4d5YpeHeA+1zV0pFpca2bvrc3YEbn0I8w+RApC13pVTUKT1QzwNvlACQnBjUBm1fym/nO+Bp\ntqY1KJgOTr1jtSsNd6VU1CnZ1wDANTOGI8EXTjMLrDlpVv78i33n3wOn3R7hCqOfhrtSKuq0BIZB\n3nTGcZ1fcOfAd1dDU2A+mr/MgdryCFcXGzTclVJRp8VrhXtyYjdj3HPHfPHYnRvy5fsGCr2gqpSK\nOi0ea3GOFGcPEeXO0YurR6DhrpSKOq2BlnuKs4e7U9uX73vzLqjbG4HKYoeGu1Iq6rS33DuNlOlO\n0Rng88Cq38LmlyNQWezQcFdKRZ1Wj4/kxITOI2W6M+NWWLQLHEnW5GOqg4a7UirqtHh8PXfJtBOx\numeOsqJTPNJwV0pFnRaPv+eLqcHc2XphtYteffdE5CIRKRGRUhE5bBIIERkuIitFZL2IbBKRi0Nf\nqlIqXrR4+9ByB6vlvv3v0HQofEXFmB7DXUQcwMPALGAccJWIjOty2F3AC8aYKcA84PehLlQpFR98\nfsOrG/b2fDE12NCTre3+LeEpKgb15rs3HSg1xuwwxrQBzwOXdznGABmBx5mAjklSSvXL/jpr6t/c\n9D4sjD1+trXVrpkOvblDdSiwJ+h5OXBKl2N+BrwpIrcBbuArIalOKRV36lus5fW+Ob0Pa6e2Tyim\nd6t26E3LvbuxSKbL86uAxcaYAuBi4FkROey9ReRmESkWkeLKSh22pJQ6XH2LB4D0lD7MjuIabG2b\ndMRMu96EezkwLOh5AYd3u9wAvABgjPkASAFyur6RMeYxY8xUY8zU3Nzc/lWslBrQ2lvuaX0Jd4cT\nUrK05R6kN+G+BhgtIkUikoR1wfS1LsfsBs4DEJGxWOGuTXOlVJ/VBVruGX0Jd9B5Zrro8btnjPGK\nyELgDcABPGmM2SIi9wDFxpjXgB8Bj4vID7C6bK4zxnTtulFKqSNqavPS6vFTWd8KQHqKs29v4MrR\nlnuQXv1qNMYsB5Z32ffToMdbgdNCW5pSKl7srWnm7Afeps1nzSkjAhl9DXd3DhzaEYbqYpPO566U\nsl3JvnrafH6+c+Zx5GemUDDIRWpSH25iAivcP1kGn2+C/InhKTSGaLgrpWxXXtMMwPWnF5GX0c/1\nUPPGW9u/L4L5y49+bBzQuWWUUrbauKeG/3xlMwkCuWl9uHGpq+k3wegLoGF/6IqLYRruSilb/Wu7\nNbDu7q+eSEJCD1P89iRrhF5UDdBwV0rZauveOgqzXXx7ZuGXfzN3DrTUWAt4xDkNd6VURLR6fTS3\ndf5o8/rZebCRUUPSQ3MSV7a1feZy+Pil0LxnjNILqkqpsFu/u5orH/0Ar7/z7S+JCYLXbzh99GE3\ntPfPcWdD0ZlQsQ42vQATvhGa941BGu5KqbBbv7sGr9/wg6+MITmwCIcx8Iu/fwJAYbYrNCfKGQ3f\n/j94chZ4mkLznjFKw10pFXa7qhpJT07ke+eN6rQu6vihGZQeaODyKUNDe0JnqtX3Hsc03JVSYVdR\n08zQQamHLXh9xuhczhgdhkkEk1xQF9/LSugFVaVU2DW0evs+ncCX4XTFfbeMhrtSKuya23x9n07g\ny3C6wNMcufNFIQ13pVTYNbb5cCdHOty15a6UUmHV3OYj1RnBS3xJgXCP45nHNdyVUmHX1ObFFdFu\nmVQwftiyNG6nI9BwV0qFXVObL7Lhnn6stX1pPrx1T+TOG0U03JVSYeXzG1q9/sheUJ00DxZ8BLlj\n43ZIpIa7UiqsmtqsBa8j2nIXgdzjIXNo3K6rquGulAqr6sb2Ba8jOM69nSsHGqsif94ooOGulAqr\nd0ut+dpH56VF/uTuHG25K6VUqG3dW8edSzcDMDovRNP69oUr2xoS2RZ/Y9413JVSYVMRWBv1p5eO\ns6dbxh2YSjgOW+8a7kqpsKlrtvrbzz1hiD0FuALhHodj3TXclVJhU9cSuJiaakOrHYJa7vF3UVXD\nXSkVNvUt1jDI9BSbZhdvX3ZPW+5KKRU6dc0eUp0OnA6bokb73JVSKrQ2V9Tyx/d24k62cU2g5Axw\nJGnLXSmlQuX1jz8H4NunjrCvCBHromocttx1mT2lVEgtfn8nr2zYS1lVI+PyM7jtvNH2FuTOhs83\n2VuDDTTclVIhtXhVGS0ePxMLsrhiyrF2l2Mt3LFnNVTvgkE2/hURYRruSqmQqGvxsPKTA5RVNfHj\nC49nwTmj7C7JMu0mK9xrdmu4K6VUXzS3+Tjl52/R7PEBcNmkKGixt8sbZ23jrN+9VxdUReQiESkR\nkVIRWXSEY+aIyFYR2SIifwltmUqpaFZR09QR7GeOyWXYYJfNFQWJ07tUewx3EXEADwOzgHHAVSIy\nrssxo4E7gNOMMScC3w9DrUqpKLW3pgWA00flcMesE2yupgvXYGv7zgPQWm9vLRHUm5b7dKDUGLPD\nGNMGPA9c3uWYm4CHjTHVAMaYA6EtUykVzfbVWuH+P7MnMDY/w+ZqunA4rWX3GvZD6Qq7q4mY3oT7\nUGBP0PPywL5gY4AxIvK+iHwoIhd190YicrOIFItIcWVlZf8qVkpFnb211uyPQzKSba7kCG54w9q2\nNdpbRwT15oKqdLPPdPM+o4GzgQLgXREZb4yp6fRJxjwGPAYwderUru+Bx+OhvLyclpaWXpSlolFK\nSgoFBQU4nTZNFKVssa+2hZy0ZJITI7iUXl843dY2juZ17024lwPDgp4XAF1XnC0HPjTGeICdIlKC\nFfZr+lJMeXk56enpFBYWItLd7xQVzYwxVFVVUV5eTlFRkd3lqAjasKeG/MwUu8s4MmeqtfXET8u9\nN90ya4DRIlIkIknAPOC1Lse8ApwDICI5WN00O/paTEtLC9nZ2RrsMUpEyM7O1r+84kzJvno+2VfP\nYHeS3aUcWUe4N9tbRwT1GO7GGC+wEHgD2Aa8YIzZIiL3iMhlgcPeAKpEZCuwEvixMaZfEyhrsMc2\n/fnFn+37rREoN51xnM2VHIWIdaeq9rl3ZoxZDizvsu+nQY8N8MPAh1IqjrQvpTdpWKbNlfTA6dKW\nuwq9mTNnHvX1iy++mJqamqMeo1Q02nOoiYyURNLtWCO1L5wua7HsOKHTD/SDz+fD4ejbqIBVq1Yd\n9fXly5cf9fVI6s/Xp+LX2l3VTCiI8lY7WP3udRV2VxExURvu/+//trB1b11I33PcsRnc/dUTj3pM\nWVkZF110Eaeccgrr169nzJgxPPPMM4wbN47rr7+eN998k4ULFzJt2jQWLFhAZWUlLpeLxx9/nBNO\nOIH9+/dzyy23sGOHdT35kUceYebMmaSlpdHQ0MDnn3/O3Llzqaurw+v18sgjj3DGGWdQWFhIcXEx\nOTk5/OpXv+LJJ58E4MYbb+T73/8+ZWVlzJo1i9NPP51Vq1YxdOhQXn31VVJTU7v9OkpLS7nllluo\nrKzE4XDw4osvsmfPHn75y1+ybNkyABYuXMjUqVO57rrrKCws7Pj6LrnkEpYuXcpHH33U8T257LLL\n2LRpE2vXruWHP/whDQ0N5OTksHjxYvLz80P1I1Ixxuc3lOyvZ8HYKJkk7GhSMmHnO7B2MZx8nd3V\nhJ12y3SjpKSEm2++mU2bNpGRkcHvf/97wBrD/d577zFv3jxuvvlmfvvb37J27Vp++ctf8t3vfheA\n733ve5x11lls3LiRdevWceKJnX+Z/OUvf+HCCy9kw4YNbNy4kcmTJ3d6fe3atTz11FOsXr2aDz/8\nkMcff5z169cD8Omnn7JgwQK2bNlCVlYWL7/88hG/hquvvpoFCxawceNGVq1a1asAbv/67rjjDtra\n2jp+QS1ZsoQ5c+bg8Xi47bbbeOmll1i7di3XX389d955Z++/sWrAqWv2YAxkp0XxSJl2l/yvtT3w\nib11REjUttx7amGH07BhwzjttNMAuOaaa3jooYcAmDt3LgANDQ2sWrWKK6+8suNzWltbAfjnP//J\nM888A4DD4SAzs/Ofq9OmTeP666/H4/Hwta997bBwf++997jiiitwu62bLmbPns27777LZZddRlFR\nUcfxJ598MmVlZd3WX19fT0VFBVdccQVghXZvtH99AHPmzOGFF15g0aJFLFmyhCVLllBSUsLmzZs5\n//zzAav7Rlvt8a222QNARrT3twPkT4RBhXEzO2TUhrudug7na3/eHrh+v5+srCw2bNjQ5/c+88wz\neeedd3j99de59tpr+fGPf8y3vvWtjtetgUfdS07+4tZuh8NBc3P3V/6P9B6JiYn4/f6O513Ho7d/\nfWAF/ZVXXsns2bMREUaPHs3HH3/MiSeeyAcffHD0L1LFjfZwz0yNgXAHa4bIOJkdUrtlurF79+6O\nAHvuuec4/fTTO72ekZFBUVERL774ImCF6caNGwE477zzeOSRRwCrZVtX1/m6wa5duxgyZAg33XQT\nN9xwA+vWrev0+plnnskrr7xCU1MTjY2NLF26lDPOOKNP9WdkZFBQUMArr7wCWH9VNDU1MWLECLZu\n3Uprayu1tbW89dZbR3yPkSNH4nA4uPfeezta9McffzyVlZUd3xuPx8OWLVv6VJsaWDrC3RUj4e7O\ngR0roWZPz8fGOA33bowdO5ann36aiRMncujQIW699dbDjvnzn//ME088waRJkzjxxBN59dVXAfjN\nb37DypUrmTBhAieffPJh4ff2228zefJkpkyZwssvv8ztt9/e6fWTTjqJ6667junTp3PKKadw4403\nMmXKlD5/Dc8++ywPPfQQEydOZObMmezbt49hw4YxZ84cJk6cyNVXX93j+86dO5c//elPzJkzB4Ck\npCReeuklfvKTnzBp0iQmT57c4yggNbDFXMs9J7Ce64qf2VpGJMjRugHCaerUqaa4uLjTvm3btjF2\n7Fhb6mlXVlbGpZdeyubNm22tI5ZFw89RRcYT7+3k3mVbWfef50f39APt/H747RTIKID5r9tdTb+I\nyFpjzNSejtOWu1Kq33YebCAz1cmgWOmWSUiAvPFxcVFVL6h2UVhYGFOt9gULFvD+++932nf77bcz\nf/58mypS8eTT/Q0U5rhja04hdy7s/tDuKsJOwz3GPfzww3aXoOJUQ6uXdburmX9ajE3v7M6Bpip4\n4Vsw/TtQeJrdFYWFdssopfplS0UtHp9h5shsu0vpm+POsbpmPlkO6/9kdzVho+GulOozr8/Pk+/v\nBGB0XrrN1fRR4Wlw63uQN25A971ruCul+mzp+gre2LKf9JRE8jOieAWmoxngNzRpuCul+uzNrftx\nJTn454/OJiEhhi6mBnPnarirL6esrIzx48cD1k1Ml156qc0VKfXlbN1bx3lj88hNT+754GjlzoHa\n3fBfedbHP39ud0UhpaNljsIYgzGGhITY+B3o9XpJTNQfqQqvuhYPFTXNfPOU4XaX8uVMvR4cTjB+\n2PxX2D2w5kyK3iT42yLY93Fo3/OYCTDrvqMe0j5v+jnnnMMHH3zA97//fR599FFaW1sZOXIkTz31\nFGlpaaxZs4bbb7+dxsZGkpOTeeutt6iqquLaa6+lsdFap/F3v/tdjyswddXQ0MBtt91GcXExIsLd\nd9/N17/+9Y754AFeeuklli1bxuLFi7nuuusYPHgw69evZ/LkySxdupQNGzaQlZUFwKhRo3j//fdJ\nSEjglltuYffu3QD8+te/7pj5Uqm+2L7PWjP1hGNi7EJqV9kj4Ss/sx5XfQZVpXZWE3LRG+42Kikp\n4amnnuKee+5h9uzZrFixArfbzS9+8Qt+9atfsWjRIubOncuSJUuYNm0adXV1pKamMmTIEP7xj3+Q\nkpLCp59+ylVXXUXXKRZ6cu+995KZmcnHH1u/2Kqrq3v8nO3bt7NixQocDgd+v5+lS5cyf/58Vq9e\nTWFhIXl5eXzzm9/kBz/4Aaeffjq7d+/mwgsvZNu2bf36/qj4ti0Q7sfHergHG4A3NkVvuPfQwg6n\nESNGMGPGDJYtW8bWrVs7WrhtbW2ceuqplJSUkJ+fz7Rp0wBrFkaAxsZGFi5cyIYNG3A4HGzfvr3P\n516xYgXPP/98x/NBgwb1+DlXXnllx7J4c+fO5Z577mH+/Pk8//zzHTM6rlixgq1bt3Z8Tl1dHfX1\n9aSnD6D/oCoiVmzdz9CsVIZmdb8KWExy51jDIj0t4IzR0T9dRG+426h9XnNjDOeffz7PPfdcp9c3\nbdrU7e3WDz74IHl5eWzcuBG/39/rRTKCGWO6fe/gfUebh/3UU0+ltLSUyspKXnnlFe666y7AmoP+\ngw8+OOKyfEr1hs9v+GjnIeZOGxZbUw70JLPA2q55HGbeZm8tIRIbVwptMmPGDN5//31KS62+uKam\nJrZv384JJ5zA3r17WbNmDWCtfOT1eqmtrSU/P5+EhASeffZZfD5fn895wQUX8Lvf/a7jeXu3TF5e\nHtu2bevodjkSEeGKK67ghz/8IWPHjiU7O7vb9+3PQiNK7ahsoNnjY/zQGFgQuy8mX2Nt6/fZW0cI\nabgfRW5uLosXL+aqq65i4sSJzJgxg08++YSkpCSWLFnCbbfdxqRJkzj//PNpaWnhu9/9Lk8//TQz\nZsxg+/btnVrUvXXXXXdRXV3N+PHjmTRpEitXrgTgvvvu49JLL+Xcc8/tcWm79nnYg5fNe+ihhygu\nLmbixImMGzeORx99tM+1KfW3zVb4nVI02OZKQsyRCJnDBtS4d53PXYWc/hwHrjl/+IBWr59XFwzA\nkVZ/OMvqe7/myAvPRwOdz10pFVKrd1Tx0c5DTC4YYF0y7dwDazoCvaBqk6eeeorf/OY3nfaddtpp\nOoWvikqV9a3MfcwaKjhveozfvHQkrhyoLLG7ipCJunA/0miRgWb+/PkDckENu7r5VHhtKq8B4CcX\nncDY/AybqwmTAdZyj6pumZSUFKqqqjQgYpQxhqqqqn4NAVXRbedB667redOG2VxJGLmywdsMbY12\nVxISUdVyLygooLy8nMrKSrtLUf2UkpJCQUGB3WWoEFtTdoictCQGxcIi2P3lzrG2jQchqe8j3aJN\nVIW70+mkqCjGluxSaoB7oXgPb2zZz4JzRtpdSni5AuH+5ythwWqI8e7hXnXLiMhFIlIiIqUisugo\nx31DRIyI9DhMRykV/d7ZXsm/v7SJpMQErps5wBteI2ZCYiocLIGWWrur+dJ6DHcRcQAPA7OAccBV\nIjKum+PSge8Bq0NdpFLKHss27QXgb7efEdtzt/dGahZ89dfW46Yqe2sJgd603KcDpcaYHcaYNuB5\n4PJujrsXuB9o6eY1pVQM+vRAA6cel83I3DS7S4kMV1C/e4zrTbgPBfYEPS8P7OsgIlOAYcaYZSGs\nTSllo4qaZjZX1DJxoN601B23NRcTdRX21hECvQn37q4qdIxVFJEE4EHgRz2+kcjNIlIsIsU6Ikap\n6Pbmln14fCb2V1zqi/TAvE0rY3/Jvd6EezkQPLi1ANgb9DwdGA+8LSJlwAzgte4uqhpjHjPGTDXG\nTM3Nze1/1UqpsDLG8NrGvRTluBmRHfvDAnst/RgYVAT+vs/oGm16E+5rgNEiUiQiScA84LX2F40x\ntcaYHGNMoTGmEPgQuMwY07cliJRSUeOtbQdYv7uG75x5nN2lRN6YC+PjgqoxxgssBN4AtgEvGGO2\niMg9InJZuAtUSkXeu59W4k5y8I2T4/CGNFcOtNaBt9XuSr6UXt3EZIxZDizvsu+nRzj27C9fllLK\nThv21DA2P4NER1TNUBIZ7RdVm6og41h7a/kS4vAnp5Q6mg17athYXssFJ+bZXYo93IHrgTE+HFLD\nXSnVyRtb9pGYIMydFkejZIK1j3Vv0nBXSg0ga8uqmVCQSWaq0+5S7NExgVhsX1TVcFdKdWj1+vi4\nopZJBVl2l2IfV6DPvTG278XRcFdKdfjRCxtp9vg4+/g4vg8lJQvEAR89Bn/9Dvi8dlfULxruSikA\nqhpaeXPLfo7LcXP6qBy7y7FPQgJMuwEkATY9D9VldlfULxruSikAnnhvJ16/nz9ce3J8DoEMdvED\ncPH91uMYvbAa5z9BpVS7TeW1nHhsJqPz0u0uJTp0DImMzb53DXelFIca2wI3Lmmwd2gfEtmwPyb7\n3TXclVL8ffM+Glq9fPOUEXaXEj3cOYDA6z+C/8qFT163u6I+0XBXSvFxRQ2ZqU4mxdPc7T1JTIZv\nPAHn3AXGD/s+truiPomqBbKVUvbYVF7LhKGZSIwvCh1y479ubT/4Xcz1vWvLXak41+LxsX1/PRO0\n1X5k7pyYm2tGw12pOLem7BAen2HqiEF2lxK9XDlQVQrblsGuVXZX0ysa7krFudU7DuFIEGaOjOMb\nl3oyuAj2b4YlV8NTs6B6l90V9UjDXak491llAyMGu0hNcthdSvS69EH4zrtwya+s53V7j358FNAL\nqkrFue376zkuN47WSe0PZyrkT7RGzUBM3LWqLXel4ljZwUY+q2zklKJsu0uJDR3TAWu4K6Wi2LJN\nVvfCJRPzba4kRrTftbruGXvr6AUNd6Xi1IG6Fv7wrx3MHJnNsVmpdpcTG5wp1vbgdnvr6AUNd6Xi\nUKvXx9cfXUV9q5cfX3i83eXEltO+D742u6vokYa7UnHo2Q92sedQM/d/fSJThuv49j5JclvhHuWT\niWm4KxVn2rx+Fq8q45SiwcyZNszucmKPM9CF5Wmyt44eaLgrFWdeWV9BeXUzt5w90u5SYpPTZW01\n3JVS0cLr8/NC8R4KBqVy9pg4Xif1y9BwV0pFm//9x3aKd1Wz8JxROgNkfyUFwr1Nw10pFQVWfXaQ\nxe+XccmEfOZNH253ObGro+XebG8dPdBwV2qAa27z8fDKUr75+GqOyUzhPy8dZ3dJsa093Ks+tbeO\nHujcMkoNYGt3HeLnr29j3e4a0lMSWfKdGQxJT7G7rNjmGmxtX7kVTrgEUqJzHnwNd6UGqIdXlvLA\nGyU4HcJ9syfwtSlDSXHqzI90WOBdAAAObElEQVRfWu4JcPJ8WPsU1O+L2nDXbhmlBqDaJg+/XrGd\ni048huI7z2fe9OEa7KEiAuMutx5H8QRivQp3EblIREpEpFREFnXz+g9FZKuIbBKRt0REl1BXykZ/\nfG8HHp9h4bmjyHQ57S5n4GmfHTKKp/7tMdxFxAE8DMwCxgFXiUjXKzLrganGmInAS8D9oS5UKdU7\nmytq+f3bnzH7pKGMHxqdXQYxzxX9U//2puU+HSg1xuwwxrQBzwOXBx9gjFlpjGkf9PkhUBDaMpVS\nvbHqs4Nc+tv3yHYncfelJ9pdzsDlCsx/31Rlbx1H0ZsLqkOBPUHPy4FTjnL8DcDfvkxRSqne8/j8\nrCk7xIc7DvHUezvJy0jmyeumaXdMOCUmQXJmVLfcexPu3d3GZro9UOQaYCpw1hFevxm4GWD4cL2J\nQqkvq7bZw7VPrGZTeS0AY/MzeOTqkyjM0WXzws6dHdV97r0J93IgeOq4AuCw1WFF5CvAncBZxpjW\n7t7IGPMY8BjA1KlTu/0FoZTqnd1VTSz4yzq2fV7H/8yewHknDGFIho5hjxhXTsy33NcAo0WkCKgA\n5gHfDD5ARKYAfwAuMsYcCHmVSqkOew418fya3Tz7wS4AfnvVFGZN0GXyIs6dAyXLrYBvHz0TRXoM\nd2OMV0QWAm8ADuBJY8wWEbkHKDbGvAY8AKQBLwYmI9ptjLksjHUrFXdqmz088d5OHv3XZ7R5/Zx6\nXDb/PXsCRdoFY4/8SVa4710Po8+3u5rD9OoOVWPMcmB5l30/DXr8lRDXpZQKaPX6WFVaxQ9e2EBN\nk4eLJxzDHbPGMmywy+7S4tuEK+Ht/4HGSrsr6ZZOP6BUlNp5sJFXN1Twx3d30tDqZdSQNJ6eP51J\nw7LsLk3BF10xUdrvruGuVJTZX9fC955bz+qdhwC4YFweZ47J5dKJ+WS5kmyuTnVIzoAEJ9RVQNMh\ncDghOd3uqjpouCsVBT6rbODlteVs3lvHhzuq8PkN/3bBGL4yLo8TjsmwuzzVHRFIPwZWP2p9IHDd\nMig83e7KAA13pSLOGMO+uha27q2jvLqZP6/exfb9DQCMyUtjztQCrptZxKghaTZXqno0+3HYtwna\nGuCte6DyEw13peJJY6uXt0sqWbq+nK1769hb29Lx2jEZKfzsq+O4cPwx5Gem2lil6rMRp1ofPo8V\n7o3RMx2BhrtSIVZcdojVOw9RdrCRTw80UFnfSmV9K20+P7npyUwvGsy1x2Yy7tgMhmalMDI3Tdcz\njXUOpzWvexTdsarhrlQIVNa38vDKUt7csq+jVZ6bnsyo3DROOW4wuWnJnH38EKYXDcaRoEE+IEXZ\nHasa7kr1QavXR22ThxaPn4qaZtbtrmb97mpW7zhEY5uXs48fwlXTh3PNjBEMcuvIlrjiztGWu1Kx\nwBjDwYY2Sg80sLemmY3lNfx1XQUNrd5Ox43MdTNrwjHceMZxjMmLnqFwKsJcOVBdZncVHTTcVdwy\nxrD7UBMH6ltpavOxv66FxlYvn3xez1uf7KeqsQ0TNL1dkiOBSybmc/KIQaQ4HWSnJTFlWJaOPVcW\ndzZUFNtdRQcNdzXgGGOobGjlYH0bLV4fZQcb2b6/gaoG66LmZ5UN7K1poaHVS5vXf9jnpyUnct7Y\nIYwY7GKQO4lRQ9IYNshFbnoy7mT9L6OOwJVjLd5hjDUG3mb6L1XFLI/PT12zhz3VzeyrbWZzRR2b\n99ZSXFZ9WNeJ0yFku5NJdiYwNCuVyROycCcnUjDIRVG2mxRnArnpyWSkOHEnJ5KUqGvHqz5y54Df\nCy01kDrI7mo03JU9mtq8HGpso8Xjo6qhjQP1rTS0evH5DX5j8Pmtj4ZWLx6fn4P1bTS0eamobmZf\nbQsHG1rx+jsvCZAgMGpIGpdNPpbj89IZkp5MSpKDIenJjB6SroGtwqtjXdUqDXcVnXx+Q6vXR6vH\nT5PHR12zh1avvyNwOwWwMfh8huqmNpo9Prw+g9fvx+s3gceGplYv1U0eDja0sr+uhc9rW6ht9vS6\nHhHISUsmxZlAYbabUUNyyE1PxuV0kJ6SSJYridF5aQwf7CI9RZeWUzZxt6+rehAYZWspoOEe1Rpb\nvdQ0e2hu81Hf4uFQYxu1zR68foM/EKz+9sA1dOyrbmqjocVq8fr84PP78ZnA1m/w+a0ujRaPryOk\n/Qb8xlDb7GFXVVPPxfWBK8lBZqqTnLRkhg12Mb1oMMdkppDtTrIuTLqTGZKRTFpyIokJQkKC4BDB\n4RDSkhJJ0HHhKha4omuWyJgL9zVlh/hXiTV/ssF0jGYwEPTYdKzyau0/8nHBoyFM4En7MSbwJuaL\nt6PF46O+xYsxViC2b/2Bz/UHzuUP7Cfw2GcMrR5/R6u363H+QFD7g/ZVNbZ2qq+3khwJpKck4nQk\n4EiQzh9ibZ0OIdnpwOlIIDnRCtQEgaFZqVw+eSiuJAcpiQmkOB1kpDpJcSaQIEJiQgIJCXS8T3sQ\nZ7mcVjg7EkgMnKt9q3dfqrjQPgVwlIx1j7lwX7+7mt+/XdoRGMIXF6YF6VjOu+t++eIlROSLVb+7\n7u/0+RL0mrU3OdEKTiu0ICFwjIgVjsFbARIShEQREkTIdid0hKME9iW0v0dg+8VzIS8jmWMzU0lJ\ncpCW7GCwO5msVCeJji+C2grl9sfgSBBSEh3a2lUq0tpb7q//G/z9P6zHiUlw1fMwfEbEy4m5cL/5\nzJHcfOZIu8tQSqnOnCnw1d/AwU+t5z4PfPQHaxk+DXellIphJ1/3xWO/H9b80bY+eB0bppRS4ZCQ\nAK7BtvXBa7grpVS4uHKg3J4pCTTclVIqXByJsH8z1FZE/NQa7kopFS7TbrK2TZFfoUnDXSmlwiWz\nwNp6QntjYG9ouCulVLg4XdZWw10ppQaQpEC4t2m4K6XUwNHRcm+O+Kk13JVSKlw6wr0x4qfWcFdK\nqXBJ0pa7UkoNPO0t97bIt9x1bhmllAoXRxKIAz78PXz84hf7z/p3GP/1sJ5aw10ppcJFBM6+A/Z/\n3Hl/SlbYT92rcBeRi4DfAA7gj8aY+7q8ngw8A5wMVAFzjTFloS1VKaVi0Fk/tuW0Pfa5i4gDeBiY\nBYwDrhKRcV0OuwGoNsaMAh4EfhHqQpVSSvVeby6oTgdKjTE7jDFtwPPA5V2OuRx4OvD4JeA80bXV\nlFLKNr0J96HAnqDn5YF93R5jjPECtUB2KApUSinVd70J9+5a4F2Xbe7NMYjIzSJSLCLFlZWVvalP\nKaVUP/Qm3MuBYUHPC4C9RzpGRBKBTOBQ1zcyxjxmjJlqjJmam5vbv4qVUkr1qDfhvgYYLSJFIpIE\nzANe63LMa8C3A4+/AfzTGHNYy10ppVRk9DgU0hjjFZGFwBtYQyGfNMZsEZF7gGJjzGvAE8CzIlKK\n1WKfF86ilVJKHV2vxrkbY5YDy7vs+2nQ4xbgytCWppRSqr/Ert4TEakEdtly8iPLAexZqrxnWlvf\nRWtdoLX1R7TWBZGtbYQxpseLlraFezQSkWJjzFS76+iO1tZ30VoXaG39Ea11QXTWprNCKqXUAKTh\nrpRSA5CGe2eP2V3AUWhtfRetdYHW1h/RWhdEYW3a566UUgOQttyVUmoA0nAPEJHbRKRERLaIyP1B\n++8QkdLAaxdGuKafiUiFiGwIfFwcDXV1qfHfRMSISE7guYjIQ4HaNonISTbUdG/g3BtE5E0ROTaK\nantARD4JnH+piGQFvWbnv7UrA//2/SIytctrtv9bE5GLAucvFZFFdtQQVMuTInJARDYH7RssIv8Q\nkU8D20F21giAMSbuP4BzgBVAcuD5kMB2HLARSAaKgM8ARwTr+hnwb93st7WuoDqGYd25vAvICey7\nGPgb1mRyM4DVNtSVEfT4e8CjUVTbBUBi4PEvgF9Ew88UGAscD7wNTI2mf2tYd8Z/BhwHJAXqGRfp\nn11QPWcCJwGbg/bdDywKPF7U/nO180Nb7pZbgfuMMa0AxpgDgf2XA88bY1qNMTuBUqz57e0WLXU9\nCPw7nWcAvRx4xlg+BLJEJD+SRRlj6oKeuoPqi4ba3jTWtNgAH2JNxNdem20/U2PMNmNMSTcvRcO/\ntd6sKRExxph3OHxixOA1LZ4GvhbRorqh4W4ZA5whIqtF5F8iMi2wvzdz2YfbwsCf8E8G/alne10i\nchlQYYzZ2OUl22sDEJGfi8ge4GqgfaqMqKgtyPVYf0lA9NXWLhrqioYaepJnjPkcILAdYnM98bNA\ntoisAI7p5qU7sb4Pg7D+VJ8GvCAix9HLeerDWNcjwL2Bc94L/C9WIIS9rl7U9h9YXQyHfVo3+yJa\nmzHmVWPMncCdInIHsBC4O1pqCxxzJ+AF/tz+aeGurTd1dfdp3eyL9BC7aKgh5sRNuBtjvnKk10Tk\nVuCvxuow+0hE/FhzRfRmLvuw1dWlxseBZYGnYa8LjlybiEzA6n/dGFhNsQBYJyLT7a6tG38BXscK\n96ioTUS+DVwKnBf4N0ckauvD9yxYRL5nMVBDT/aLSL4x5vNAV9+BHj8jzLRbxvIKcC6AiIzBumhz\nEGue+nkikiwiRcBo4KNIFdWlP/gKoP3qvK11GWM+NsYMMcYUGmMKsf7znWSM2Reo7VuBkSkzgNr2\nP1cjRURGBz29DPgk8DgaarsI+AlwmTGmKeglW3+mRxENdfVmTQm7Ba9p8W3gSH8JRY7dV3Sj4QMr\nzP+EFZ7rgHODXrsT60p9CTArwnU9C3wMbML6x5MfDXV1U2cZX4yWEeDhQG0fEzTyIoL1vBz4WW4C\n/g8YGkW1lWL1H28IfDwaDT9TrMZDOdAK7AfeiIa6gmq4GNgeqONOO2oIquU54HPAE/ie3YC1ZvRb\nwKeB7WA7azTG6B2qSik1EGm3jFJKDUAa7kopNQBpuCul1ACk4a6UUgOQhrtSSg1AGu5KKTUAabgr\npdQApOGulFID0P8H71GUpQoFq2AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xdafcd68>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(thresholds,precisions,label='precision_curve')\n",
    "plt.plot(thresholds,recalls,label='recall_curve')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 可视化精准率-召回率曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGvlJREFUeJzt3XucFfV9//HXm+V+EeTiFXCNggrE\nS1xRY+ItNqC/FHpJDFSTmFpJ0mhsY5OatDHWpk1q2qS1khqSqEkaRZPmws8fERNjoiZBAUUFLBEJ\nCCK6XAUWWHb5/P6YYTwsy+6R3dk5u/t+Ph7nwZmZ7875nAHOe+c7c75fRQRmZmYAPYouwMzMKodD\nwczMMg4FMzPLOBTMzCzjUDAzs4xDwczMMg4F69YkXSHpoTLa3SHpcx1RU14k3S3pC+nzCyWtLbom\nqzw9iy7ArEgR8T3ge2W0+2gHlGNWOJ8pWKcnqcv8ctOV3ot1Tg4Fq1iSVkn6jKRlkjZLuktS331d\nH5L+VtJ64K60/XskLZa0RdJvJJ1asq9Rkn4oqVbSRkm3p+uvkvR4+lySvirpNUlbJT0raUK6Let6\nSZevkbRC0iZJcyQdU7ItJH1U0gtp3TMlqYX3GZI+LukF4IV03cmSfpbuf7mky0va95P0b5JWp3U+\nLqlfuu37ktan6x+VNL59/jasu3AoWKW7ApgEnACMBf4+XX8UMBQ4Dpgh6W3AncBHgGHA14E5kvpI\nqgIeAFYD1cCxwOxmXuvdwPnp6wwB3g9sbNpI0sXAF4HLgaPT/Tbd33uAs4DT0naTWnmffwScDYyT\nNAD4GXAPcAQwHfhayQf8vwJnAm9Pj8Gngb3ptp8CY9Kfe4oyusbMSjkUrNLdHhFrImIT8E8kH5CQ\nfAh+PiJ2R8RO4Brg6xHxREQ0RsS3gd3AOcBE4BjgUxGxIyJ2RcTjzbzWHmAQcDKgiHg+Il5ppt0V\nwJ0R8VRE7AY+A5wrqbqkzZciYktEvAQ8Apzeyvv8YkRsSt/Le4BVEXFXRDRExFPA/wDvldQD+HPg\n+oh4OX2vv0nrICLujIht6fLNwGmSBrfy2mYZh4JVujUlz1eTfLgD1EbErpJtxwE3pF1HWyRtAUal\n7UcBqyOioaUXiohfALcDM4FXJc2SdFgzTY9Ja9n3c9tJziiOLWmzvuR5HTAQQNJSSdvTxzsP8j6P\nA85u8l6uIDk7Gg70BV5sWpSkKklfkvSipNeBVemm4S29b7NSDgWrdKNKno8G1qXPmw7vuwb4p4gY\nUvLoHxH3pttGl3MRNyJui4gzgfEk3UifaqbZOpIPbgDS7p5hwMtl7H98RAxMH4+VbmryXn7V5L0M\njIiPARuAXSTdaU39GTAVuAQYTNJVBnDQ6xlmTTkUrNJ9XNJISUOBzwL3HaTdN4CPSjo7vWA8QNL/\nkTQIeBJ4BfhSur6vpPOa7kDSWenP9wJ2kHz4NjbzWvcAH5Z0uqQ+wD8DT0TEqja/28QDwFhJH5DU\nK32cJemUiNhLcu3kK5KOSc8Ozk3rGETSZbYR6J/WZfamOBSs0t0DPASsTB9faK5RRCwkua5wO7AZ\nWAFclW5rBP4QOBF4CVhLchG5qcNIwmUzSffQRpKLuk1f62HgcyT9/K+Q/NY+7dDeXrPvZRvJRe9p\nJGcl64F/AfqkTf4GeA5YAGxKt/UAvpPW/TKwDJjfXjVZ9yFPsmOVStIq4C8i4udF12LWXfhMwczM\nMg4FMzPLuPvIzMwyPlMwM7NMpxt8a/jw4VFdXV10GWZmncqiRYs2RMSI1tp1ulCorq5m4cKFRZdh\nZtapSFrdeit3H5mZWQmHgpmZZRwKZmaWcSiYmVnGoWBmZpncQkHSnem0hksOsl2SbkunNHw2nTnL\nzMwKlOeZwt3A5Ba2X0oybeAYYAbwXznWYmZmZcjtewoR8WiT6Qmbmgp8J5JxNuZLGiLp6INMf9hm\nC1Zt4rHf1eaxa+tuJM4fM5ya6qFFV2LW7or88tqx7D8F4dp03QGhIGkGydkEo0ePPqQXe2r1Zv7z\nkRWH9LNmpSLgtodf4Ozjh3LtxSfyjhOHI3lyM+saigyF5v4XNTs6X0TMAmYB1NTUHNIIfh+54AQ+\nckFzMxiavTl19Q3c++QaZj36Ih/41pOcNnIwH7/oRC455Uh69HA4WOdW5N1Ha9l//t2RvDH/rlnF\n6t+7J1e/43ge/fRFfPFP3srmuj3M+O4iLv2Px/jJ4pdpaNxbdIlmh6zIUJgDfDC9C+kcYGte1xPM\n8tCnZxXTJ47mFzdcwL+//3T2RnD97MW86yu/YvaTL1Hf4HCwzie3+RQk3QtcCAwHXgU+D/QCiIg7\nlHTC3k5yh1Id8OF0nt0W1dTUhAfEs0q0d2/w0LJXmfnICp57eSvDB/bhqMF9Wv/BMn3o3GreVzOq\n9YZmzZC0KCJqWmuX591H01vZHsDH83p9s47Wo4eYPOEoJo0/ksde2MB9C9ewq76xzfvdsnMPi1Zv\n5sXaHe1QpVnLOt3Q2WaVThLnjx3B+WNbHbq+LLc9/AKLVm9m2lk+S7D8eZgLswrW0LiXe598iXeO\nGU718AFFl2PdgM8UzCrYL5fX8srWXVx38Rhe27Yrl9c4rG8v+vaqymXf1vk4FMwq2M+WvQrAZ3/0\nHJ/9UT6v0bdXDy4cewSTJhzJxScfyeB+vfJ5IesUHApmFewvLzqBt44cnNv+A/jd+m08tGw9Dy5d\nT88e4twThjF5wlH8wbgjOWJQ39xe2ypTbrek5sW3pJq1v717g8VrtzBv6XrmLVnPqo11SHDm6MOZ\nNP4oJo0/itHD+hddprVBubekOhTMbD8Rwe9e3c6DS9Yzb+l6lr3yOgCnHH0Yk8YfyeQJR3HSkYM8\n3lMn41Aws3axZlMd85au58El61n00mYi4Lhh/Zk8/ijeMmL/O6JGDOrDRScd4cCoQA4FM2t3r23b\nxc+Wvcq8pa/y2xc3sKfxwM+Ppf8wiQF9fLmy0hT+jWYz63qOGNSXK84+jivOPo7tuxt4feeebNuV\n33yCQf16ORA6OX95zcwOycA+PTlmSD+OGdKPuvpGVm7YwZTTjim6LGsjh4KZtdmcZ9YhwR+eenTR\npVgb+TzPzNrsgWfW0bdnFZ/7yZJs3aC+vfj0pJM44jB/16EzcSiYWZudNmoIvXu+zuqNddm6lRt2\n8MrWnXz3z8/2jHSdiEPBzNrsq+8//YB19z75Ep/54XN88/GVzDjfU+F2Fr6mYGa5mHbWKCaNP5Iv\nz1vOkpe3Fl2OlcmhYGa5kMSX/uRUhg3owydmP83OdphwyPLn7iMzy83hA3rzlctP44pvPcFbb55H\njy78TedpE0dxy9QJRZfRZg4FM8vV208czh1XnsniNVuKLiU335u/mle25jPfRUdzKJhZ7vaNtNoV\nRQR3/3oVo4d2jVFkfU3BzKwNNmyvZ+eeRkYd3q/oUtqFzxTMzNpgzebkuxlbdu7h8Rc2HNI+egjO\nGH04/XoXPy2qQ8HMrA3qdid3Vf37z19o037++pKxXH/JmPYoqU0cCmZmbXDeicN44Lp3sHPPod1y\n27g3mDZrfjtXdegcCmZmbSCJCcce+jzaW+uS4ccH9q2Mj2NfaDYzK9C23UkoDHIomJnZ9t0NAAyq\nkMmJHApmZgXavisJBXcfmZkZ29IzhYE+UzAzs23pmYKvKZiZWdZ9NKhvr4IrSTgUzMwKtD29+6hb\ndB9JmixpuaQVkm5sZvtoSY9IelrSs5Iuy7MeM7NKs31XAxL0r4AhLiDHUJBUBcwELgXGAdMljWvS\n7O+B+yPiDGAa8LW86jEzq0TbdjcwsE9PVCFzTeR5pjARWBERKyOiHpgNTG3SJoDD0ueDgXU51mNm\nVnG27WqomO8oQL6hcCywpmR5bbqu1M3AlZLWAnOB65rbkaQZkhZKWlhbW5tHrWZmhdi+q6FivqMA\n+YZCc+dC0WR5OnB3RIwELgO+K+mAmiJiVkTURETNiBEjcijVzKwY23c3VMydR5BvKKwFRpUsj+TA\n7qGrgfsBIuK3QF9geI41mZlVlH3XFCpFnqGwABgj6XhJvUkuJM9p0uYl4F0Akk4hCQX3D5lZt7F9\n157u0X0UEQ3AtcA84HmSu4yWSrpF0pS02Q3ANZKeAe4FroqIpl1MZmZdVqVdaM61koiYS3IBuXTd\nTSXPlwHn5VmDmVkl296Nuo/MzKwFDY17qatv7B7dR2Zm1rLX03GPhvTrHncfmZlZC7bU1QMwuL9D\nwcys29u6MxkMb0i/3gVX8gaHgplZQfaFwmHuPjIzs32hMNihYGZmWfeRrymYmdmWOp8pmJlZauvO\nPfTvXUWvqsr5KK6cSszMupmtO/dU1HcUwKFgZlaYLXV7KurOI3AomJkV5vWdeyrqegI4FMzMCrN1\n556KuvMIHApmZoXZsrPeZwpmZgYRwZY6dx+ZmRlQV9/I7oa9DB3Qp+hS9uNQMDMrwKYdyQipwwZW\nzmB44FAwMyvExn2hMMChYGbW7W3asRuAoQ4FMzPbuH3fmYKvKZiZdXv7rikM9TUFMzPbuKOe3j17\nMKB3VdGl7MehYGZWgI3b6xk2oDeSii5lPw4FM7MCbNqxu+IuMoNDwcysEJt21DsUzMwssXFHfcV9\nRwEcCmZmhUjOFCrrdlRwKJiZdbhdexqpq2+suCEuwKFgZtbhKnWIC3AomJl1uNptyRAXIwa5+8jM\nrNvrtqEgabKk5ZJWSLrxIG0ul7RM0lJJ9+RZj5lZJajkUOiZ144lVQEzgT8A1gILJM2JiGUlbcYA\nnwHOi4jNko7Iqx4zs0qxLxT+e/5qelWV97t5r6oeTJ84OvfvNuQWCsBEYEVErASQNBuYCiwraXMN\nMDMiNgNExGs51mNmVhGOOCw5Q5j5yItv6ueGDujN9Imj8ygpk2coHAusKVleC5zdpM1YAEm/BqqA\nmyPiwaY7kjQDmAEwenS+B8TMLG/TJ45m2lmjym5fu203E//5YRr3Ro5VJfK8ptDcKE9N31FPYAxw\nITAd+KakIQf8UMSsiKiJiJoRI0a0e6FmZh1NUtmPZj9Nc5JnKKwFSqNwJLCumTY/iYg9EfF7YDlJ\nSJiZWQHyDIUFwBhJx0vqDUwD5jRp82PgIgBJw0m6k1bmWJOZmbUgt1CIiAbgWmAe8Dxwf0QslXSL\npClps3nARknLgEeAT0XExrxqMjOzluV5oZmImAvMbbLuppLnAXwyfZiZWcH8jWYzM8s4FMzMLNNi\n95Gk/8uBt5FmImLKwbaZmVnn09o1hX/tkCrMzKwitBgKEfGrjirEzMyK11r30XO03H10artXZGZm\nhWmt++g9HVKFmZlVhNa6j1Z3VCFmZla8sm5JlXSOpAWStkuql9Qo6fW8izMzs45V7vcUbicZxfQF\noB/wF8B/5lWUmZkVo+xhLiJihaSqiGgE7pL0mxzrMjOzApQbCnXpSKeLJd0KvAIMyK8sMzMrQrnd\nRx9I214L7CCZJ+FP8yrKzMyKUe6ZwgagPiJ2Af8gqQrok19ZZmZWhHLPFB4G+pcs9wN+3v7lmJlZ\nkcoNhb4RsX3fQvq8fwvtzcysEyo3FHZIetu+BUlnAjvzKcnMzIpS7jWFvwK+L2ldunw08P58SjIz\ns6KUFQoRsUDSycBJgID/jYg9uVZmZmYdrtxhLvoDfwtcHxHPAdWSPFiemVkXU+41hbuAeuDcdHkt\n8IVcKjIzs8KUGwonRMStwB6AiNhJ0o1kZmZdSLmhUC+pH+mEO5JOAHbnVpWZmRWi1QvNkgTcATwI\njJL0PeA84Kp8SzMzs47WaihEREi6Hng3cA5Jt9H1EbEh7+LMzKxjlfs9hfnAWyLi/+VZjJmZFavc\nULgI+Iik1SSjpIrkJOLU3CozM7MOV24oXJprFWZmVhHK/Ubz6rwLMTOz4pV7S6qZmXUDDgUzM8s4\nFMzMLJNrKEiaLGm5pBWSbmyh3XslhaSaPOsxM7OW5RYK6TzOM0nuXBoHTJc0rpl2g4BPAE/kVYuZ\nmZUnzzOFicCKiFgZEfXAbGBqM+3+EbgV2JVjLWZmVoY8Q+FYYE3J8tp0XUbSGcCoiHigpR1JmiFp\noaSFtbW17V+pmZkB+YZCc0NrR7ZR6gF8FbihtR1FxKyIqImImhEjRrRjiWZmVirPUFgLjCpZHgms\nK1keBEwAfilpFclge3N8sdnMrDh5hsICYIyk4yX1BqYBc/ZtjIitETE8Iqojoppk0L0pEbEwx5rM\nzKwFuYVCRDQA1wLzgOeB+yNiqaRbJE3J63XNzOzQlTsg3iGJiLnA3CbrbjpI2wvzrMXMzFrnbzSb\nmVnGoWBmZhmHgpmZZRwKZmaWcSiYmVnGoWBmZhmHgpmZZRwKZmaWcSiYmVnGoWBmZhmHgpmZZRwK\nZmaWcSiYmVnGoWBmZhmHgpmZZRwKZmaWcSiYmVnGoWBmZhmHgpmZZRwKZmaWcSiYmVnGoWBmZhmH\ngpmZZRwKZmaWcSiYmVnGoWBmZhmHgpmZZRwKZmaWcSiYmVnGoWBmZhmHgpmZZXINBUmTJS2XtELS\njc1s/6SkZZKelfSwpOPyrMfMzFqWWyhIqgJmApcC44DpksY1afY0UBMRpwI/AG7Nqx4zM2tdnmcK\nE4EVEbEyIuqB2cDU0gYR8UhE1KWL84GROdZjZmatyDMUjgXWlCyvTdcdzNXAT5vbIGmGpIWSFtbW\n1rZjiWZmVirPUFAz66LZhtKVQA3w5ea2R8SsiKiJiJoRI0a0Y4lmZlaqZ477XguMKlkeCaxr2kjS\nJcDfARdExO4c6zEzs1bkeaawABgj6XhJvYFpwJzSBpLOAL4OTImI13KsxczMypBbKEREA3AtMA94\nHrg/IpZKukXSlLTZl4GBwPclLZY05yC7MzOzDpBn9xERMReY22TdTSXPL8nz9c3M7M3xN5rNzCzj\nUDAzs4xDwczMMg4FMzPLOBTMzCzjUDAzs4xDwczMMg4FMzPLOBTMzCzjUDAzs4xDwczMMg4FMzPL\nOBTMzCzjUDAzs4xDwczMMg4FMzPLOBTMzCzjUDAzs4xDwczMMg4FMzPLOBTMzCzjUDAzs4xDwczM\nMg4FMzPLOBTMzCzjUDAzs4xDwczMMg4FMzPLOBTMzCzjUDAzs4xDwcyswvWpquKytx7F6KH9c3+t\nXENB0mRJyyWtkHRjM9v7SLov3f6EpOo86zEz64wG9+/F1644k/PHjsj9tXILBUlVwEzgUmAcMF3S\nuCbNrgY2R8SJwFeBf8mrHjMza12eZwoTgRURsTIi6oHZwNQmbaYC306f/wB4lyTlWJOZmbUgz1A4\nFlhTsrw2Xddsm4hoALYCw3KsyczMWpBnKDT3G38cQhskzZC0UNLC2tradinOzMwOlGcorAVGlSyP\nBNYdrI2knsBgYFPTHUXErIioiYiaESPyv9BiZtZd5RkKC4Axko6X1BuYBsxp0mYO8KH0+XuBX0TE\nAWcKZmbWMXrmteOIaJB0LTAPqALujIilkm4BFkbEHOBbwHclrSA5Q5iWVz1mZta63EIBICLmAnOb\nrLup5Pku4H151mBmZuVTZ+utkVQLrC66jnYwHNhQdBEVxMfjDT4Wb/Cx2F9bjsdxEdHqRdlOFwpd\nhaSFEVFTdB2VwsfjDT4Wb/Cx2F9HHA+PfWRmZhmHgpmZZRwKxZlVdAEVxsfjDT4Wb/Cx2F/ux8PX\nFMzMLOMzBTMzyzgUzMws41DIWRkTDX1S0jJJz0p6WNJxRdTZEVo7FiXt3ispJHXpWxHLOR6SLk//\nfSyVdE9H19hRyvh/MlrSI5KeTv+vXFZEnR1B0p2SXpO05CDbJem29Fg9K+lt7VpARPiR04NkeI8X\ngbcAvYFngHFN2lwE9E+ffwy4r+i6izoWabtBwKPAfKCm6LoL/rcxBngaODxdPqLougs8FrOAj6XP\nxwGriq47x+NxPvA2YMlBtl8G/JRklOlzgCfa8/V9ppCvVicaiohHIqIuXZxPMppsV1TOpEsA/wjc\nCuzqyOIKUM7xuAaYGRGbASLitQ6usaOUcywCOCx9PpgDR1zuMiLiUZoZLbrEVOA7kZgPDJF0dHu9\nvkMhX+VMNFTqapLfALqiVo+FpDOAURHxQEcWVpBy/m2MBcZK+rWk+ZImd1h1HaucY3EzcKWktSTj\nqV3XMaVVpDf7ufKm5DognpU3iRCApCuBGuCCXCsqTovHQlIPknm6r+qoggpWzr+NniRdSBeSnEE+\nJmlCRGzJubaOVs6xmA7cHRH/JulcktGVJ0TE3vzLqzhlf64cCp8p5KuciYaQdAnwd8CUiNjdQbV1\ntNaOxSBgAvBLSatI+krndOGLzeVOQvWTiNgTEb8HlpOERFdTzrG4GrgfICJ+C/QlGRyuOyrrc+VQ\nORTy1epEQ2mXyddJAqGr9hlDK8ciIrZGxPCIqI6IapLrK1MiYmEx5eaunEmofkxyIwKShpN0J63s\n0Co7RjnH4iXgXQCSTiEJhe46N+8c4IPpXUjnAFsj4pX22rm7j3IU5U009GVgIPB9SQAvRcSUworO\nSZnHotso83jMA94taRnQCHwqIjYWV3U+yjwWNwDfkPTXJF0lV0V6K05XI+leki7D4ek1lM8DvQAi\n4g6SayqXASuAOuDD7fr6XfS4mpnZIXD3kZmZZRwKZmaWcSiYmVnGoWBmZhmHgpmZZRwKZu1I0m9a\n2T5X0pCOqsfszfItqWYHIakqIhqLrsOsI/lMwbolSdWS/lfSt9Mx6X8gqb+kVZJukvQ48D5JJ0h6\nUNIiSY9JOjn9+SMl/UjSM+nj7en67emfR0t6VNJiSUskvTNdvyr9dvK+uTSWpI+/KqnreUnfSOdQ\neEhSv0IOknVLDgXrzk4CZkXEqcDrwF+m63dFxDsiYjbJOP7XRcSZwN8AX0vb3Ab8KiJOIxn7fmmT\nff8ZMC8iTgdOAxaXbpR0Jsk3Uc8mGefpmnTIE0jGN5oZEeOBLcCfttcbNmuNh7mw7mxNRPw6ff7f\nwCfS5/cBSBoIvJ03hiAB6JP+eTHwQYC0i2lrk30vAO6U1Av4cUQsbrL9HcCPImJH+lo/BN5JMq7N\n70vaLwKq2/Aezd4UnylYd9b0gtq+5R3pnz2ALRFxesnjlLJ2nEyUcj7wMskwzx9s0qS54Y/3KR0p\ntxH/8mYdyKFg3dnodGx+SMbrf7x0Y0S8Dvxe0vsgmxv3tHTzwyTTpyKpStJhpT+rZK7t1yLiG8C3\nSLqYSj0K/FF6HWMA8MfAY+331swOjUPBurPngQ9JehYYCvxXM22uAK6W9AzJdYN900ReD1wk6TmS\nLp7xTX7uQmCxpKdJrgn8R+nGiHgKuBt4EngC+GZEPN0O78msTXxLqnVLkqqBByJiQsGlmFUUnymY\nmVnGZwpmZpbxmYKZmWUcCmZmlnEomJlZxqFgZmYZh4KZmWX+P6j0NSrVOZZFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xdd5fd68>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(precisions,recalls)\n",
    "plt.xlabel('precision')\n",
    "plt.ylabel('recall')\n",
    "plt.title('precision-recall')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用sklearn封装的方法绘制曲线：\n",
    "- 注意返回的threasholds,precisions,recalls的shape值\n",
    "- sklearn会自动设置合适的thresholds的步长"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "precisions,recalls,thresholds=precision_recall_curve(y_test,decisoin_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(77L,)\n",
      "(77L,)\n",
      "(76L,)\n"
     ]
    }
   ],
   "source": [
    "print precisions.shape\n",
    "print recalls.shape\n",
    "print thresholds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8VNX5x/HPk8lKCIRAWAMk7Hsg\nhF22AopUQVBUrIqiUEXUaktrS1vXX6u1tdWKO4KgIqKgiCiiQNmRRUB2AgYMIJCwk4Qkk/P74w4Y\nQkiGMDN3ZvK8X695zXLP3Hm4mXy5Offcc8UYg1JKqeASYncBSimlPE/DXSmlgpCGu1JKBSENd6WU\nCkIa7kopFYQ03JVSKghpuCulVBDScFdKqSCk4a6UUkEo1K4PrlGjhklMTLTr45VSKiCtW7cu0xgT\nX1Y728I9MTGRtWvX2vXxSikVkERkrzvttFtGKaWCkIa7UkoFIQ13pZQKQhruSikVhDTclVIqCJUZ\n7iLytogcFpHNl1guIvKSiKSJyCYRSfF8mUoppS6HO3vuU4CBpSy/Fmjquo0BXr3yspRSSl2JMse5\nG2OWiEhiKU2GAFONdb2+VSISKyJ1jDEHPVTjhfathp1fQL/HQcQrH6FUec1c+yM/Hs22uwzl5/q1\nrEVy/VivfoYnTmKqB/xY5HmG67WLwl1ExmDt3dOgQYPyfdrBjbDs39B5DFSpW751KOUFOw+dYvxH\nmwDd71Clq1klMiDCvaSvcYlX3TbGvAG8AZCamlq+K3PXaWfdH9yo4a78yvur9xHuCGHVn/oRFx1u\ndzmqgvPEaJkMoH6R5wnAAQ+st2S12gACBzd57SOUuly5+U5mrc9gYJvaGuzKL3gi3OcAd7pGzXQF\nTnitvx0gojLUaGrtuSvlJz7fdJCTuQWM6FzO7kalPKzMbhkRmQ70AWqISAbwOBAGYIx5DZgHDALS\ngGzgbm8Ve16dZNj2Gbzeq/R2YZXgmr9BPR2dGchOny1g/7EcQh1CWEgIoQ4h1CGEO0IIdYQQERpC\nmMPeUzamf7uPRjWi6dooztY6lDrHndEyI8pYboAHPFaRO1JHQd4ZMIWlt9u/Hj66G+5bbu3xq4A0\nasoavv3haKltQkOEqHAH0eGhxEWHU71yOJFhDiJCQ4gMcxAZFkJEqINw138E1n8IQpgjhPDQECqF\nO2haM4ZK4Q4iXO8LD7XahTtCkFKOkO48dIq1e48xYVDLUtsp5Uu2Tfl7RRp2t25l2bsCJg+C+X+C\nwS95vy7lcUfP5LEm/SjDOtSjd/N48p2GAmch+c5C63FhIWfzC8nJd5Kd5+TM2QKOnsnjaHYemafz\nOJvvJDffSW5BIbn5zvPvu1wxkaEkVKtEo/hoGtWIJi46nLjocOJjIpiz4QDhjhBu7JjghS2gVPkE\nZri7q2F36PEQLH8Rml9r3VRAWbrrCMbAyO6JHhs6ZowhzxXyeQXWfxTHs/P5IfM0Zwus/yzOFjit\nx67b8ew8Mo7lsCnjOPO+P4gp9v/D4OS6eiBV+ZXgDneAvhMg7RuY8yDc/SVExHj+M8IiIbKq59er\nWLT9MNWjw2lbz3PbV0SICHUQEQpEWK/VqhJJ89rufTechYYTOfkcPZPHwRM5pB0+zTWta3usPqU8\nIfjDPTQChr0Bb/SBlzt65zPEAePWQPXG3ll/BVVYaFiyK5PezeIJCfGfvmxHiJzvlmlSszI9m5Z5\nxTOlfC74wx2gVmsY9SUc2OD5dacvgy2z4NRBDXcP+9/OIxw9k0ef5hqeSl2uihHuAPU6WjdPq9XG\nCveCs55fdwWWm+/k8TlbaFQjmoFttMtDqctVccLdWxxh1r0zz946gsx/F+5i39Fs3h/dhYhQh93l\nKBVwNNyvVKjriNyaSXDmiHc+IyIGWlwPjorx49p56BRvLNnDjSkJdG9cw+5ylApIFSMtvCk6HkLC\nIG2BdfOW1kNh2Js//6UQQHLznazfd4wVaVnsPHSKiDAHUWEhRIU5iAx3UCkslKhw63lUeCjvrd5L\n5YhQJvyypd2lKxWwNNyvVOWaMD4N8k577zO+nwlfPwGFBXDj2xDq3+OpnYWGrQdOsiwtkxW7M/n2\nh6OcLSjEESI0qhGNs9CQneckJ9+65RVcfKbxv4Yn67hxpa6AhrsnRMVaN2+56hFwRMD8P8LMu2D4\n5J+7g/yAMYYfMs+wfHcWy3dlsnJPFidy8gFoXiuG27o04KomNeicFEdM5MV/eTgLjRX0edbZpI4Q\noW5slK//GUoFFQ33QNFtrNUlM+93MOMOuHmqdfKUTQ6fzGX57kyWp2WxIi2TAydyAagXG8U1rWvR\no0kNujWuTs2Ysmt0hAiVI0KpHKFfR6U8RX+bAknn0RDigLmPwKT+UC3J2qv3wayXzkLDou2HWZaW\nyfK0THYdtrqhYiuF0b1xdcY2rsFVTWrQsHolnTxLKT+g4R5oUkdBaBSsmgjb5kBcI6+HuzGG383c\nyOzv9hMZFkKnxDhu6phAjyY1aFWnil+dPaqUsmi4B6L2I6zb3+v75OSpF7/Zxezv9vNQv6Y80Lex\njjtXKgBouAey0AgoyPXqR3zy3X7+8/UubkxJ4JH+TbXLRakAYe/la9SVCY3y6p77mvSj/P6jTXRt\nFMffh7XVYFcqgOieeyALi4KN02HzRx5ftQGSnYVsDoOwQyHI3z204qg46PYAdLoXwit5aKVKqeI0\n3APZ1c/AvhVeWfWZswVMXbmX3s3iaV23iudWfGADLPgLrPgv9HwUOt5t65BOpYKVhnsga3a1dfOC\njJ9O8o+lS2nYPoXW7ep4duV7V8Civ8GXj1lXyer5W0i5069OzFIq0GmfuyrRiWzrDNOqUV6Yy6Zh\nd7hrLoz8DKolWidmvZQCaydDgc6uqZQn6J67KtGezDOAl8L9nKRekNgT9iyGRf8Hc38Dy16Aqx61\nQr+omi0hRud1V8pdGu7qIpv3n+DpuVtpXbcKLep44ZqzRYlA477QqA+kff1zyBcXFg0DnoTUeyBE\n/+BUqiwa7uoCB47nMGrKGmKjwnj7rk6EOXwUpCLQdAA06Q8HN0J+zs/LCvNh6QtW983WT2HwS9aZ\nuUqpS9JwV+edys1n1JQ15OQ5mXl/N2pVsWEUiwjUbX/x64k94btpMH8CvNoD+j0OncfoXrxSl6C/\nGeq8pz7bStrh07xyewotantw+KMniFgjasaugoY94Ms/wJRBkLXb7sqU8ksa7gqAnDwnn39/kOGp\n9enZNN7uci6taj341Uy44VU4vBVe7Q5r37a7KqX8joa7AmDxjsNk5zm53tNj2r1BBNrfBmNXW3vx\ncx+FtG/srkopv6LhrgCY+/1BqkeH0zkpzu5S3FelDtzyrjVMctYYOHnQ7oqU8hsa7gqAVbuz6NO8\nJqG+Gh3jKeGVYPgUyM+Gj+8FZ4HdFSnlFwLsN1l5S76zkJjIAB08Fd8cfvkC7F0G/3vW7mqU8gtu\nhbuIDBSRHSKSJiKPlbC8gYgsEpHvRGSTiAzyfKnKm4zdBVyp9iOg/e2w5J+wb7Xd1ShluzLDXUQc\nwETgWqAVMEJEWhVr9mfgQ2NMB+BW4BVPF6q8p7DQUFhoCPjp2gf9AyKrwJq37K5EKdu5s+feGUgz\nxuwxxuQBHwBDirUxwLmB0VWBA54rUXnb1JXpnMlz0r5+rN2lXJnwaGhzI2z7DHJP2l2NUrZyJ9zr\nAT8WeZ7heq2oJ4DbRSQDmAc86JHqlNfty8rmuS930Kd5PIOT69pdzpVLvg0KcqxpCpSqwNwJ95L+\nWC/eRTsCmGKMSQAGAdNE5KJ1i8gYEVkrImuPHDly+dUqjyosNPzh402Ehgh/Gxokl9FLSIXqTawr\nVClVgbkT7hlA/SLPE7i42+Ue4EMAY8xKIBKoUXxFxpg3jDGpxpjU+Hg/Pguygvg2/Sgr92Txh2tb\nUDc2yu5yPOPcCU57l8PGGXZXo5Rt3An3NUBTEUkSkXCsA6ZzirXZB/QDEJGWWOGuu+Z+bnlaJo4Q\nYUj7IOiOKarLfdZc8bN/rVMTqAqrzHA3xhQA44D5wDasUTFbROQpERnsavZbYLSIbASmA3cZYwJ+\ndF2wW56WSbuEqsREevGCHHYIj4bbZkLTq2HuI7DiZbsrUsrn3DprxRgzD+tAadHX/lrk8Vagh2dL\nU552+mwBa9KPsmp3Fit2Z/H9/hM8+IsmdpflHWGR1tQEs0bDVxMg7wz0/j2BP95TKfcE6CmJyh05\neU7W7T3Gyj2ZrNydxcaMEzgLDeGOENo3iOU3/ZsyplcQX/QiNBxunARhlWDx3+DsSej/JDj0a6+C\nn37Lg8jZAicb9h1nxe4sVu7JYsO+4+Q5C3GECMkJVbmvdyO6NapBx4bViAp32F2ubzhCYchEq6tm\n5cvWgdYhE6FWa7srU8qrNNyDxHur9/LM3G3k5DsJEWhTryp390ika+PqdEqMo3JEBf5Rh4TAoOeh\nYXeYNx5e7w29xsNVj1h790oFoQr8Gx8cCgsNz325ndeX7KFn0xrc2S2RzklxVI0KsoOkV0oE2gyD\npN7WVZwW/8060emGiVC3g93VKeVxGu4B6myBkxFvrGLDj8cpNHB71wY8cX3rwJuy19eiq8ONb0Hr\nYdZImjf7QfcHoc8frYOwSgUJTYIAUOAsvOi191btY/0+K9gnDGrJ00PaaLBfjhaD4IHV1glPy/8D\nr/eEnGN2V6WUx+ieux84kZNPxrFsMo7lsP9YjnV/3PX8eA4ncvJJqhFNckIs7RKq0qpOFSYuSgNg\n6qjO9GqmZ/uWS1QsDHkZGveFj0bBnv9B6xvsrkopj9Bw9zJjDMeyrfD+ObhzLgjzU2cvvHpQVJiD\nhGpR1KsWRYcGsVSNCmPHT6dZnpbJ7O/2n2/3yQM9An8mR3/Q7FpA4MgOuytRymM03D3EWWj4astP\npGdl/7zX7QrznHznBW1jIkKpVy2KhGpRdG1UnXqxUefDPKFaJapVCrvkJF4/nchlY8ZxAA12Twmv\nBLENIFPDXQUPDXcPeembXbz4zS4AYiuFkVAtikbx0fRqFn9ReF/JSJbaVSOpXbW2p8pW58Q31z13\nFVQ03D0gr6CQ97/dR+9m8Uz8VUrFHlMeqOJbwO6FsHmWNWRSqQCnwys8YMHWQxw5dZa7uidqsAeq\nrvdDnWT46G74+F4dOaMCnoa7ByzZeYRqlcJ01Eogq1IXRn0FfSfAltnwSjdI+8buqpQqNw13D9i0\n/wRtE2JxhOiMgwHNEWrNHHnv1xBRBd4dBp//1ppRUqkAo+F+hfKdhew8dIo2dauU3VgFhrod4Nf/\ng27jYM0keO0q2L/O7qqUuiwa7lco41gOzkJDZFgFmWWxogiLgmv+D0Z+Bs58mD4CnAVlv08pP6Hh\nfoVeW7wbgBlrfrS5EuUVST1h4LNw+hDs1j54FTg03K9Qn+bWQdTfD2xucyXKa5peDVFxsOF9uytR\nym0a7lfoeE4+AK21zz14hYZD2+GwY54OkVQBQ8P9Cn2+6SCJ1SvROL6y3aUob2p/GzjzYPPHdlei\nlFs03MvBWWhYsPUQd0xazbK0TK5PrnvJuWBUkKiTDNWbwvbP7a5EKbfo6ZSXIfP0WWas+ZH3V+9j\n//EcaleJ5NEBzYL7ItPKIgJNB1hDI/OyrcnGlPJjGu5u2pRxnJteW0leQSHdG1fnL9e1pH/LWnqB\njIqkST9Y9Yp1ke2mA+yuRqlSabi7advBk+QVFPLx/d3o2DDO7nKUHRr2gNBIa1oCDXfl53S3001Z\nZ/IAaFWnqs2VKNuERVkBv+NzKMizuxqlSqXh7qbcPCciEBWuZ6JWaJ3HwPF9sOR5uytRqlQa7m44\neiaP7/efwKEjYlTzgZA8Apb+C/avt7sapS5Jw70UP53I5em5W+nx7EIW7zzCTR0T7C5J+YOBz0Ll\nmvDJ/ZCfa3c1SpVID6i6GGP46WQum/efZMuBE2zef4IlOzNxGsOQ9nUZ26cxTWrG2F2m8gdRsTD4\nZXjvRlj8dxjwpN0VKXWRChnuhYWGvUezXSFuhfmWAyc56jpoKgKNakRza+f6jO7ZiPpxOqZZFdO0\nP3S4A1a8ZPXDV61nd0VKXSCow/3M2QIqhTvOnz2a7yzk6blbmbV+P6fPWtO3hjmEZrVi6N+yJm3q\nVaV13Sq0qF2FaL1cnipLtwfgu2mwaz6kjrK7GqUuEJQJlnEsmxcW7GT2d/tpEFeJIcl16d+qFs99\nuZ3laVkMS6lHl6Q4WtetSrNaMYSH6qEHVQ7xLSC2Iez4UsNd+R23wl1EBgIvAg7gLWPMsyW0uRl4\nAjDARmPMbR6s0y3ZeQX8e8FO3lmxFwR+1aUB6ZnZvLwojZcWphEaIvxzeLIeGFWeIQLNr4V1U3RK\nAuV3ygx3EXEAE4EBQAawRkTmGGO2FmnTFPgj0MMYc0xEanqr4NK8tfQH3lz6A8M7JvDIgGbUjY0C\n4PDJXBZsO0TrulVpXz/WjtJUsGo2EFa/Bl+Mh6r1QRzQfgRU1R0IZS939tw7A2nGmD0AIvIBMATY\nWqTNaGCiMeYYgDHmsKcLdcf3+0/QOD6a54cnX/B6zSqR/KpLQztKUsGuYQ+olgjfvfvza6YQ+vzB\ntpKUAvfCvR5Q9BpyGUCXYm2aAYjIcqyumyeMMV8WX5GIjAHGADRo0KA89V7S9xknWLT9MDd3qu/R\n9SpVqtBweGjDz8+fqQX52fbVo5SLO0cSSzot0xR7Hgo0BfoAI4C3ROSi/g9jzBvGmFRjTGp8fPzl\n1npJxhge+XAD8TER/P4avdyd8jGRn2+hkVBw1u6KlHIr3DOAorvDCcCBEtp8aozJN8b8AOzACnuf\nOHTyLGmHT/PrXo2IrRTuq49V6mKh4eDUcFf2cyfc1wBNRSRJRMKBW4E5xdp8AvQFEJEaWN00ezxZ\n6KXsOXKa8R9tBKB1PZ2xUdksNBJOHgRT/I9bpXyrzHA3xhQA44D5wDbgQ2PMFhF5SkQGu5rNB7JE\nZCuwCBhvjMnyVtHnvLlkDwP/s5QN+47z+PWtSG1YzdsfqVTpWl4PO7+ALx+DwkK7q1EVmFvj3I0x\n84B5xV77a5HHBnjUdfOZ57/aQYcGsfz3tg7UjIn05UcrVbKr/8+6X/UK5ByDIRPBEWZvTapCCrgz\nVNekH2Xqyr2M69uEvIJCujWursGu/EdICFzzN6gUBwufgdyTMHyydaEPpXwo4MI941g2n208wJeb\nD1Kjcjg3tNcJm5SfEYFe4yGqGnz+O3j3RhgxHSL1mJDynYCbVGVohwRG90yiZkwkH4zpSmKNaLtL\nUqpkne6FG9+CH1fDO4OtbhqlfESMTUf1U1NTzdq1a8v9fmPM+dkelfJrO+fDjNuhVmu44xNrPnil\nyklE1hljUstqF3B77udosKuA0ewauHka/LQZpg2FnON2V6QqgIANd6UCSvOBcMu78NP38O4wyD1h\nd0UqyGm4K+UrzQfCLdPg4CZrDz77qN0VqSCm4a6ULzW/Fm6eagX8y51g04d6NqvyCg13pXytxSAY\ns9iaKnjWaJh2A2TttrkoFWw03JWyQ+02cM9X8Mt/wf718Eo3WPI8FOTZXZkKEhruStklxGGNhR+3\nxtqbX/gMvN4T9q60uzIVBDTclbJbTG0YPgVum2ldi3XyQJg1BtKX6+RjqtwCbvoBpYJWs6shcRUs\nfhbWTIJNM6zrsrYdDu1ugZot7K5QBZCAPUNVqaCWdwa2z7MCfvdCME6o3c4K+bY3WXv7qkJy9wxV\nDXel/N3pw7B5lhX0B9aDhEBSb0i+FVpcBxGV7a5Q+ZCGu1LBKHOXNTZ+0ww4vhfCKkGLX1p79I36\ngkN7WoOdhrtSwcwY+PFbK+S3zLJmnKyWBPd+A9HV7a5OeVHQTxymVIUmAg26wHUvwG93WqNtTmTA\n3N/oGa8K0HBXKvCFhkProfCLCbBtjtVtoyo8DXelgkX3h6B+V5g33tqLVxWahrtSwSLEAUNfhcIC\n+OR+PQGqgtNwVyqYxDWCAU/CD0tgn05jUJFpuCsVbJJ6Wfenf7K3DmUrDXelgk1UnHWvFwOp0DTc\nlQo2UdWs++wse+tQttJwVyrYOEIhsqo1bYGqsDTclQpGdVNg/VTYNNPuSpRNNNyVCkbDp0D9LjDr\nXljxst3VKBtouCsVjKJi4faPoeVg+GoCfPVnHfdewWi4KxWswiKtPfhO98KK/8In94Ez3+6qlI/o\n/KBKBbMQBwz6p3Vxj4XPQH423DzNmnhMBTW39txFZKCI7BCRNBF5rJR2N4mIEZEyp6NUSvmICPQa\nD/0eh22fWTcV9MoMdxFxABOBa4FWwAgRaVVCuxjgIWC1p4tUSnlA94egZmuY/yfrQtwqqLmz594Z\nSDPG7DHG5AEfAENKaPc08A8g14P1KaU8xREKg56HEz/C0n/ZXY3yMnfCvR7wY5HnGa7XzhORDkB9\nY8xcD9amlPK0xB7Q9mZY8RJk7ba7GuVF7oR7SUdezl/qRURCgH8Dvy1zRSJjRGStiKw9cuSI+1Uq\npTzn6qfBEQ5L/ml3JcqL3An3DKB+kecJwIEiz2OANsBiEUkHugJzSjqoaox5wxiTaoxJjY+PL3/V\nSqnyi6kNTfrB3mV2V6K8yJ1wXwM0FZEkEQkHbgXmnFtojDlhjKlhjEk0xiQCq4DBxhi9+rVS/qp+\nFzi+D04etLsS5SVlhrsxpgAYB8wHtgEfGmO2iMhTIjLY2wUqpbygfhfrPuNbe+tQXuPWSUzGmHnA\nvGKv/fUSbftceVlKKa+q3Q4cEbDnf9CqpMFvKtDp9ANKVUSh4dCkP6ydBFOug32r7K5IeZiGu1IV\n1U1vw8Bn4cgOePsaePdG2L/e7qqUh2i4K1VRhUVC1/vh4Q0w4Ckr2N/sC9Nvg582212dukIa7kpV\ndOHR0ONh+M0m6PtnSF8Gr/WAD0dae/UqIGm4K6UsETHQezz8ZqM10Vja1/BKV5h9H5w8UPb7lV/R\ncFdKXSiqGvziz/DwJug2DjbPgv+mWme05uvUUYFCw10pVbLo6tZUBQ+shia/gIVPw8ROsHUOGFP2\n+5WtNNyVUqWLS4Jb3oU7P4XwyvDhHfDO9XrQ1c9puCul3NOoD/x6qXVlp0Ob4fWeMPdROJNld2Wq\nBBruSin3OUKh82h4cL11bdZ1U+DFZJg3Hg5vt7s6VYSGu1Lq8lWKsy78cd8yaDHICvlXulhnu26Z\nrRfi9gNibDowkpqaatau1YkjlQoKZzLhu2mw9m1rtsnKtaDjXZAyEqrWK/Ptyn0iss4YU+Z1qjXc\nlVKeU+i0xseveQt2LQAJsfbsU++x+uylpGv/qMvhbri7NSukUkq5JcQBza6xbsfSYe1kWD8Vtn0G\n1ZtYId/pXmviMuVV2ueulPKOaokw4El4dBsMfQOi4mD+H+GbJ+2urELQcFdKeVdYJCTfAvcugJQ7\nYfVrkJlmd1VBT8NdKeU7v/gLhEbBVxPsriToabgrpXynck3o/XvY+aV14FV5jYa7Usq3utwHcY3g\nyz9Zo2uUV2i4K6V8KzQcev4OMnfA4W12VxO0NNyVUr5Xt4N1f0SnLPAWDXellO9VbwIhoXB4q92V\nBC0Nd6WU74WGWwGv3TJeo+GulLJHzZawf51OGewlGu5KKXt0uhdyT8A718Hpw3ZXE3Q03JVS9ki8\nCm770JqDZvIgvQi3h2m4K6Xs06g33D4LTv0Ek6+FY3vtrihoaLgrpezVsJt1fdacY9YefNZuuysK\nChruSin7JXSEkXMhPxvmPGh3NUFBw10p5R/qtINWQ3Tsu4douCul/EdcktU9k3Pc7koCnlvhLiID\nRWSHiKSJyGMlLH9URLaKyCYR+UZEGnq+VKVU0KuWaN0f1wOrV6rMcBcRBzARuBZoBYwQkVbFmn0H\npBpj2gEfAf/wdKFKqQqgWpJ1f/QHe+sIAu7suXcG0owxe4wxecAHwJCiDYwxi4wx2a6nq4AEz5ap\nlKoQqjeG8Mqw/h0wxu5qApo74V4P+LHI8wzXa5dyD/DFlRSllKqgwqOh/xOweyF8967d1QQ0d8Jd\nSnitxP9SReR2IBV4/hLLx4jIWhFZe+TIEferVEpVHKn3QMMeMH+CnrV6BdwJ9wygfpHnCcBFW1xE\n+gMTgMHGmLMlrcgY84YxJtUYkxofH1+eepVSwS4kBAb/F5x5MPcR7Z4pJ3fCfQ3QVESSRCQcuBWY\nU7SBiHQAXscKdp0BSCl1Zao3hl/82brW6mcPw4ENGvKXKbSsBsaYAhEZB8wHHMDbxpgtIvIUsNYY\nMwerG6YyMFNEAPYZYwZ7sW6lVLDrej8c2QYbp1sHWONbQPKt0PZmqFraYT8FIMam/w1TU1PN2rVr\nbflspVQAyTkGW2bDxg/gx9WAQFIvK+hbXg8RMXZX6FMiss4Yk1pmOw13pVTAyNoNmz6ETR9YUwWH\nVbICvt0t0KgPhDhsLtD7NNyVUsHLGGsvfuMHsGWWddGPmDrQ9iZIHgG1WttdoddouCulKob8XNg1\n3wr6XV9BYQHcPA1aBedhP3fDvcwDqr6Un59PRkYGubm5dpcSkCIjI0lISCAsLMzuUpTynbBIazbJ\nVkPgTCa82dc6ABuk4e4uvwr3jIwMYmJiSExMxDXqRrnJGENWVhYZGRkkJSXZXY5S9oiuAa2HwcqX\nIfsoVIqzuyLb+NWUv7m5uVSvXl2DvRxEhOrVq+tfPUq1GWZ1zWybU3bbIOZX4Q5osF8B3XZKAbXb\nQVxj2DzL7kps5XfhHqy6d+9e6vJBgwZx/LheoECpKyYCrYdC+lJrjHwFpeFeDk6n87Lfs2LFilKX\nz5s3j9jY2PKWpJQqqukAMIXwwxK7K7GNhnsx6enptGjRgpEjR9KuXTtuuukmsrOzSUxM5KmnnuKq\nq65i5syZ7N69m4EDB9KxY0d69uzJ9u3bATh06BBDhw4lOTmZ5OTk86FeuXJlAA4ePEivXr1o3749\nbdq0YenSpQAkJiaSmZkJwAsvvECbNm1o06YN//nPf87X1bJlS0aPHk3r1q25+uqrycnJ8fXmUSow\n1OsI4TGwe5HdldjGr0bLFPXkZ1vYeuCkR9fZqm4VHr++7JMbduzYwaRJk+jRowejRo3ilVdeAayh\nhsuWLQOgX79+vPbaazRt2pRUVcHlAAAMjElEQVTVq1czduxYFi5cyEMPPUTv3r2ZPXs2TqeT06dP\nX7Du999/n2uuuYYJEybgdDrJzs6+YPm6deuYPHkyq1evxhhDly5d6N27N9WqVWPXrl1Mnz6dN998\nk5tvvpmPP/6Y22+/3UNbR6kg4gizpijYvdDuSmzjt+Fup/r169OjRw8Abr/9dl566SUAbrnlFgBO\nnz7NihUrGD58+Pn3nD1rzXK8cOFCpk6dCoDD4aBq1aoXrLtTp06MGjWK/Px8brjhBtq3b3/B8mXL\nljF06FCio6MBGDZsGEuXLmXw4MEkJSWdb9+xY0fS09M9/C9XKog07gs7PoflL0KzgVCjmdUfX0H4\nbbi7s4ftLcVHnZx7fi5wCwsLiY2NZcOGDZe97l69erFkyRI+//xz7rjjDsaPH8+dd955fnlpZwxH\nREScf+xwOLRbRqnStBoCayfDgr9at5g61vwz524xte2szuu0z70E+/btY+XKlQBMnz6dq6666oLl\nVapUISkpiZkzZwJWIG/cuBGwumteffVVwDrwevLkhV1Le/fupWbNmowePZp77rmH9evXX7C8V69e\nfPLJJ2RnZ3PmzBlmz55Nz549vfLvVCqoVa4JY1fAwxvh+hehQVfYOR9m/xr+1RwmdoUvHoMdX8LZ\nU3ZX63Ea7iVo2bIl77zzDu3atePo0aPcf//9F7V57733mDRpEsnJybRu3ZpPP/0UgBdffJFFixbR\ntm1bOnbsyJYtWy543+LFi2nfvj0dOnTg448/5uGHH75geUpKCnfddRedO3emS5cu3HvvvXTo0MF7\n/1ilgl21ROh4FwyfAuN3w6+XQP8nrT33dZNh+i3wXCK8PRAWPwv7VoEz396aPcCvJg7btm0bLVu2\ntKWec9LT07nuuuvYvHmzrXWUlz9sQ6UCRn6uNbvknkWwZ7F1xSeMNdIm8aqfu3Dim/tNf31AThym\nlFI+FRYJjXpbN7Dmo0lfagX97kWw8wvr9cSeMPIzvwl4d2i4F5OYmBiwe+1KqStUKe7nGSbBuiDI\nt29aE5HtXwcJZe4w+w3tc1dKqUuplgi9fw+OCOsKUAFEw10ppUoTWRWaD7Su+OQssLsat2m4K6VU\nWdoOhzNH4IfFdlfiNg13pZQqS9OrIaIqbPnE7krcpuHuA+np6bRp0wawxrlfd911NleklLosoRHQ\nqJc1gsam4eOXS8O9FMYYCgsL7S5DKeUPGvWFkxmQlWZ3JW7RcC/m3NS6Y8eOJSUlhWnTptGtWzdS\nUlIYPnz4+Vke16xZQ/fu3UlOTqZz586cOnWK9PR0evbsSUpKCikpKWXO4a6UCiCN+1r3ATKNsP+O\nc//iMfjpe8+us3ZbuPbZMpvt2LGDyZMn89RTTzFs2DC+/vproqOjee6553jhhRd47LHHuOWWW5gx\nYwadOnXi5MmTREVFUbNmTRYsWEBkZCS7du1ixIgRFD8LVykVoOIaQWxD2PwRJPaAmq38+qQm/w13\nGzVs2JCuXbsyd+5ctm7den7637y8PLp168aOHTuoU6cOnTp1AqyJxADOnDnDuHHj2LBhAw6Hg507\nd9r2b1BKeUHHu+CbJ+HV7lClHjTpB036W1MURFYt482+5b/h7sYetrecm9rXGMOAAQOYPn36Bcs3\nbdpU4sWo//3vf1OrVi02btxIYWEhkZGRPqlXKeUjPR+F5Fsh7WvrtuVTWD8VQkKhfhcr6Jv0t3oJ\nbN6r1z73UnTt2pXly5eTlmYdQMnOzmbnzp20aNGCAwcOsGbNGgBOnTpFQUEBJ06coE6dOoSEhDBt\n2rRyXWtVKeXnqtSFlDvh5qnw+91w9xfQ/SE4e9Laq3+9J/yrBXzyAGyZbdtFuv13z90PxMfHM2XK\nFEaMGHH+SkvPPPMMzZo1Y8aMGTz44IPk5OQQFRXF119/zdixY7nxxhuZOXMmffv2Pf8XgFIqSDnC\noGF369b/cTj1E6R9A2kLYPtnsOFdEAckdIKm/aHJAKjdDkK8v1+tU/4GGd2GSvkJZwHsXwu7Flhh\nf9C6oA/RNWHg36HtTeVarU75q5RSdnKEWld/atAV+v0FTh/+ea8+po7XP96tvw1EZKCI7BCRNBF5\nrITlESIyw7V8tYgkerpQpZQKaJVrQvsRcNPb1lBKLysz3EXEAUwErgVaASNEpFWxZvcAx4wxTYB/\nA895ulCllFLuc2fPvTOQZozZY4zJAz4AhhRrMwR4x/X4I6CflDRW0A12HQMIBrrtlFLnuBPu9YAf\nizzPcL1WYhtjTAFwAqh+ucVERkaSlZWlIVUOxhiysrJ0bL1SCnDvgGpJe+DF09edNojIGGAMQIMG\nDS56Q0JCAhkZGRw5csSNslRxkZGRJCQk2F2GUsoPuBPuGUD9Is8TgAOXaJMhIqFAVeBo8RUZY94A\n3gBrKGTx5WFhYSQlJblXuVJKqUtyp1tmDdBURJJEJBy4FZhTrM0cYKTr8U3AQqN9K0opZZsy99yN\nMQUiMg6YDziAt40xW0TkKWCtMWYOMAmYJiJpWHvst3qzaKWUUqVz6yQmY8w8YF6x1/5a5HEuMNyz\npSmllCov26YfEJEjwF6gBpBpSxFl09oun7/WBVpbefhrXVBxa2tojIkvq5Ft4X6+AJG17syTYAet\n7fL5a12gtZWHv9YFWltZdMpfpZQKQhruSikVhPwh3N+wu4BSaG2Xz1/rAq2tPPy1LtDaSmV7n7tS\nSinP84c9d6WUUh7m83B3zfu+wXVLF5ENl2iXLiLfu9qtLamNF2p7QkT2F6lv0CXalTq/vZdqe15E\ntovIJhGZLSKxl2jnk+3mr3P8i0h9EVkkIttEZIuIPFxCmz4icqLIz/mvJa3LS/WV+vMRy0uu7bZJ\nRFJ8UFPzIttig4icFJHfFGvjs20mIm+LyGER2VzktTgRWSAiu1z31S7x3pGuNrtEZGRJbbxQm1/9\nbp5njLHtBvwL+OsllqUDNXxczxPA78po4wB2A42AcGAj0MoHtV0NhLoePwc8Z9d2c2cbAGOB11yP\nbwVm+OhnWAdIcT2OAXaWUFsfYK4vv1vu/nyAQcAXWJPxdQVW+7g+B/AT1lhqW7YZ0AtIATYXee0f\nwGOux4+V9P0H4oA9rvtqrsfVfFCb3/xuFr3Z1i3jmu/9ZmC6XTWUkzvz23ucMeYrY02nDLAKawI3\nu/h0jv/LYYw5aIxZ73p8CtjGxVNU+7MhwFRjWQXEioj3r8n2s37AbmPMXh9+5gWMMUu4eOLBot+n\nd4AbSnjrNcACY8xRY8wxYAEw0Nu1+dnv5nl29rn3BA4ZY3ZdYrkBvhKRda6pgn1lnOvPq7cv8aef\nO/Pbe9sorL27kvhiu/lsjv8r4eoK6gCsLmFxNxHZKCJfiEhrH5ZV1s/H7u/XrVx6h8uubQZQyxhz\nEKz/wIGaJbSxe9uB/b+b53nlAtki8jVQu4RFE4wxn7oej6D0vfYexpgDIlITWCAi213/a3qtNuBV\n4GmsH8LTWN1Go4qvooT3emTIkTvbTUQmAAXAe5dYjVe2W/FSS3itXHP8e4uIVAY+Bn5jjDlZbPF6\nrG6H067jKp8ATX1UWlk/H9u2m1izvg4G/ljCYju3mbvs/s75w+/meV4Jd2NM/9KWizXn+zCgYynr\nOOC6Pywis7G6Aq54Q5RVW5Ea3wTmlrDInfnty8WN7TYSuA7oZ1ydeCWswyvbrRiPzfHvDSIShhXs\n7xljZhVfXjTsjTHzROQVEalhjPH6PCVu/Hy89v1yw7XAemPMoeIL7NxmLodEpI4x5qCrm+pwCW0y\nsI4NnJMALPZBbf70u3meXd0y/YHtxpiMkhaKSLSIxJx7jHXAYnNJbT2pWN/m0Et8pjvz23ujtoHA\nH4DBxpjsS7Tx1Xbz2zn+Xf36k4BtxpgXLtGm9rn+fxHpjPV7kOWD2tz5+cwB7nSNmukKnDjXHeED\nl/xr2q5tVkTR79NI4NMS2swHrhaRaq4u1atdr3mVn/1u/sxXR26LHTWeAtxX7LW6wDzX40ZYIzA2\nAluwuiV8Udc04HtgE9aXqU7x2lzPB2GNwtjtw9rSsPoTN7hurxWvzZfbraRtADyF9QUHiARmuur+\nFmjko+10Fdaf4puKbKtBwH3nvnPAONf22Yh1AKy7j2or8edTrDYBJrq26/dAqo9qq4QV1lWLvGbL\nNsP6D+YgkI+1N34P1vGab4Bdrvs4V9tU4K0i7x3l+s6lAXf7qDa/+t08d9MzVJVSKgjpGapKKRWE\nNNyVUioIabgrpVQQ0nBXSqkgpOGulFJBSMNdKaWCkIa7UkoFIQ13pZQKQv8PVu0vdOGN+fAAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xf160550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(thresholds,precisions[:-1],label='precision')\n",
    "plt.plot(thresholds,recalls[:-1],label='recall')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAF0dJREFUeJzt3XuYXXV97/H3J5MbgdwnQO4BEoQI\nIjIElIpUbUvQQ+rRYlKoYpEUBXuep+g5UFtqqVQp55TnoaAltkDxCBT12KYaTS3QAiqYAeSWC4SQ\ny+QCk/t1kkzme/7YK7jZ2cmsSfbsNfPL5/U8+3n2Xus3a333D+aT3/zW3uuniMDMzNLSp+gCzMys\n9hzuZmYJcribmSXI4W5mliCHu5lZghzuZmYJcrhbEiRdLunfc7T7e0l/Xo+auouk+yR9NXt+kaSW\nomuynqdv0QWY1UJEfAf4To5219ShHLPCeeRuPYakZAYbKb0X650c7tbtJC2XdKOkhZI2SbpX0sD9\nUwqS/pekdcC9WfuPSvqVpM2Sfi7pXWXHGi/p/0lqlbRB0p3Z9islPZk9l6TbJb0paYukFySdke17\na0oje321pKWSNkqaK2lM2b6QdI2kV7O675KkQ7zPkHStpFeBV7Ntp0n6aXb8JZIuK2t/jKT/I2lF\nVueTko7J9n1X0rps++OS3lmb/xp2tHC4W71cDvwOcApwKvBn2fYTgRHARGC2pPcA9wB/BIwE7gbm\nShogqQH4IbACmASMBR6qcq7fBi7MzjMM+CSwobKRpA8CXwMuA0Znx6083keBc4Gzsna/08n7/F3g\nPGCqpGOBnwIPAMcDs4BvlAX1/wbOAd6X9cH/BDqyfT8GpmQ/9yw5ppzMyjncrV7ujIhVEbERuIVS\n0EEpzP4iInZHxC7gauDuiHg6IvZFxD8Bu4HzgWnAGOBLEbEjItoi4skq59oLDAZOAxQRiyJibZV2\nlwP3RMSzEbEbuBF4r6RJZW2+HhGbI2Il8Bjw7k7e59ciYmP2Xj4KLI+IeyOiPSKeBb4PfEJSH+AP\ngf8REauz9/rzrA4i4p6I2Ja9/gpwlqShnZzb7C0Od6uXVWXPV1AKaYDWiGgr2zcRuD6bktksaTMw\nPms/HlgREe2HOlFEPArcCdwFvCFpjqQhVZqOyWrZ/3PbKY3wx5a1WVf2fCdwHICklyVtzx7vP8j7\nnAicV/FeLqf010ojMBB4rbIoSQ2Svi7pNUlbgeXZrsZDvW+zcg53q5fxZc8nAGuy55W3JV0F3BIR\nw8oegyLiwWzfhDwXKyPijog4B3gnpemZL1VptoZSAAOQTaOMBFbnOP47I+K47PFE+a6K9/JfFe/l\nuIj4HLAeaKM0TVXp94EZwIeBoZSmoAAOOt9vVsnhbvVyraRxkkYAfwr880HafQu4RtJ52YXRYyV9\nRNJg4JfAWuDr2faBki6oPICkc7Of7wfsoBSi+6qc6wHgM5LeLWkA8NfA0xGx/IjfbckPgVMl/YGk\nftnjXEmnR0QHpWsLfytpTDZaf29Wx2BKU1EbgEFZXWZd4nC3enkA+HdgWfb4arVGEdFMad79TmAT\nsBS4Mtu3D/hvwGRgJdBC6WJppSGU/pHYRGnaZQOli5eV53oE+HNK8+BrKY2iZx7e26v6XrZRurg7\nk9JfCeuAW4EBWZMvAi8CC4CN2b4+wP1Z3auBhcBTtarJjh7yYh3W3SQtBz4bEf9RdC1mRwuP3M3M\nEuRwNzNLkKdlzMwS5JG7mVmCCru5UWNjY0yaNKmo05uZ9UrPPPPM+ogY1Vm7wsJ90qRJNDc3F3V6\nM7NeSdKKzlt5WsbMLEkOdzOzBDnczcwS5HA3M0uQw93MLEGdhruke7Llyl46yH5JuiNbquyFbCUd\nMzMrUJ6R+33AxYfYP53ScmBTgNnAN4+8LDMzOxKdhntEPE7pdqQHMwO4P0qeAoZJGl2rAjuzYsMO\nbv3JYvZ1+DYKZmb71WLOfSxvX1qshbcvU/YWSbMlNUtqbm1tPeITb9yxhyvvXcCDv1zJuq1tnf+A\nmdlRohbhXm3pr6rD6IiYExFNEdE0alSn3549pLa9+5h9fzOrN+/iHz7VxNhhxxzR8czMUlKLcG/h\n7etjjuPX62N2i46O4PqHn6d5xSZuv+zdNE0a0Z2nMzPrdWoR7nOBT2Wfmjkf2BIRa2tw3IO69SeL\n+dGLa7lx+ml85F11m943M+s1Or1xmKQHgYuARkktwF8A/QAi4u+BecAllNa63Al8pruKBXh08Rvc\n/fgyAOY8voxvPbGsO093WM6dNIJvXnFO0WWY2VGs03CPiFmd7A/g2ppV1InJowbz6fdOpL0Hfjrm\nZ0vXs3zDTk4cOrDoUszsKFfYLX8P14SRg/jLGWcUXcYBnni1lYcWrOIDp47iy5ecXnQ5ZnaU8+0H\nauCVN7bx+f/7LFOOP447f/9s+ja4W82sWL1u5N7TtG7bzWfuXcCOPe188tzx/Py1DUWX1C369+3D\nBac00r+v/+Ey6w0c7kfoRy+sYfXmXQD85b8tLLia7nX/H07jwlOP7PsJZlYfDvcjdMX5Eznv5JF0\nRM+7wFsLz6/awp/+4EXeP6WRCyY3Fl2OmeXkcD9CfRv6cProIUWX0S3Wb9/N3z3azLjhx3DHzLNp\n6FPty8hm1hM53K2qvfs6uPY7z7Jxxx6+/7n3MfzY/kWXZGZd4HC3qu545FWefn0jxw8ewG3zlxRd\nDmOGDeSW3z2TPv7rwSwXh7tVNfSYfpw1fhgAm3ftLayO7W17ea11Byc1HltYDWa9kcPdqvrs+0/m\ns+8/udAaIoKr73+GlRt3csfMsz1qN+sCf2jZeqz7fr6c/1j0BjdOP50zxw0tuhyzXsUjd+uRXlq9\nha/NW8w5E4fz2+884a3vElRqkDhhyAAkj+rNyjncrUe6bf4S9uzr4JkVm/iNWx87ZNvbP3kWHzt7\nXJ0qM+sdHO7WI90w/TQ+cubB79Xf1r6PW360iGMH9OV9p/jLVWaVHO7WI50+esghvxz25R+8yO72\nDu7+g7M4YYhvsWxWyRdUrdf50Qtr+c7TK/mjC0/monccX3Q5Zj2SR+7Wq6zevIsbvv8CfbOPRd42\nf3HBFVm9/MbkUbz3lJFFl9FrONytV1m+fgd7OzoA+McnXy+4GquX9o5g0dptDvcucLhbr3LB5EYW\n/9X0osuwOnpzWxvTbnmEpknDiy6lV/Gcu5n1aE++uh6AC6d4LYGucLibWY/2xKvrGXlsf6Ymemvt\n7uJpGTPr0X75+kY279rLe77600LrGDKwHz/4/PsYedyAQuvIy+FuZj3adR+czOK1WwutYfG6bTz9\n+ka27253uJuZ1cKsaROKLoG/y9Y36E1fmPOcu5lZJ9ZtbWP4oH4M7NdQdCm5OdzNzDqxbksbJw49\npugyusThbmbWibVb2jhxSO+Ya9/P4W5m1om1W3YxephH7mZmydi1Zx+bdu5lzNDeczEVHO5mZoe0\nbmsbAKM9525mlo612RKPo4clOHKXdLGkJZKWSrqhyv4Jkh6T9JykFyRdUvtSzczqb//6vWNSG7lL\nagDuAqYDU4FZkqZWNPsz4OGIOBuYCXyj1oWamRVhzeZsWibBkfs0YGlELIuIPcBDwIyKNgHsv6vP\nUGBN7Uo0MyvO6s07OX7wAAb07T1fYIJ84T4WWFX2uiXbVu4rwBWSWoB5wBeqHUjSbEnNkppbW1sP\no1wzs/pas7mNMb3sY5CQL9xVZVtUvJ4F3BcR44BLgG9LOuDYETEnIpoiomnUKN+b2cx6vtWbdzF2\neJrh3gKML3s9jgOnXa4CHgaIiF8AA4HGWhRoZlaUjo5g9aZdjEs03BcAUySdJKk/pQumcyvarAQ+\nBCDpdErh7nkXM+vV3ty2mz37Ohg3fFDRpXRZp+EeEe3AdcB8YBGlT8W8LOlmSZdmza4Hrpb0PPAg\ncGVEVE7dmJn1Ki2bdgIwvheO3HPdzz0i5lG6UFq+7aay5wuBC2pbmplZsVZl4Z7kyN3M7Gi1amPp\nC0ypzrmbmR2VVm7cyQlDBvSqRTr2c7ibmR3Eyo07mTCi903JgMPdzOygVm3cyYQRxxZdxmFxuJuZ\nVdG2dx/rtrZ55G5mlpJVG3cSAZMaHe5mZslYvqH0MciJIz0tY2aWjOXrdwBwksPdzCwdyzfsYNig\nfgwd1K/oUg6Lw93MrIrX1+/otVMy4HA3M6vq9fU7OKXR4W5mloyde9pZu6WNk0c53M3MkrGstXQx\n9eRRxxVcyeFzuJuZVVi2fn+4e+RuZpaM197cTh/BJF9QNTNLx9LW7YwfMahX3g1yP4e7mVmFpW9s\nZ3Ivnm8Hh7uZ2du07+vg9fU7mHyCw93MLBkrN+5kz74Oj9zNzFLyyhvbAHjHiYMLruTIONzNzMos\nWbcdCSYf75G7mVkyXnljGxNGDGJQ/75Fl3JEHO5mZmUWr9vKqSf07ikZcLibmb2lbe8+lm/YyWm9\nfL4dHO5mZm955Y1t7OsIpo4eUnQpR8zhbmaWWbR2KwCnO9zNzNKxcM1Wju3fwIQRvXNR7HIOdzOz\nzKK12zht9BD69FHRpRwxh7uZGdDREby8ZgtnjOn9UzLgcDczA0r3cN+xZx9njB1adCk1kSvcJV0s\naYmkpZJuOEibyyQtlPSypAdqW6aZWfd6afUWAM4cl0a4d/oVLEkNwF3AbwEtwAJJcyNiYVmbKcCN\nwAURsUnS8d1VsJlZd3gxC/eXVm9l8dpt3Xaefg19+OBpx3NM/+69V3ye79dOA5ZGxDIASQ8BM4CF\nZW2uBu6KiE0AEfFmrQs1M+tO29vaAfjid5/v9nPd9ol38XtN47v1HHnCfSywqux1C3BeRZtTAST9\nDGgAvhIRP6k8kKTZwGyACRMmHE69Zmbd4qsfO4NrLjqlW8+xccduPv7NX7C7vaNbzwP5wr3aZ4Ki\nynGmABcB44AnJJ0REZvf9kMRc4A5AE1NTZXHMDMrTL+GPpzU2L1rph47oH7L9uW5oNoClP/9MA5Y\nU6XNv0bE3oh4HVhCKezNzKwAecJ9ATBF0kmS+gMzgbkVbf4F+E0ASY2UpmmW1bJQMzPLr9Nwj4h2\n4DpgPrAIeDgiXpZ0s6RLs2bzgQ2SFgKPAV+KiA3dVbSZmR1arrvRR8Q8YF7FtpvKngfwJ9nDzMwK\n5m+ompklyOFuZpYgh7uZWYIc7mZmCXK4m5klyOFuZpYgh7uZWYIc7mZmCXK4m5klyOFuZpYgh7uZ\nWYIc7mZmCXK4m5klyOFuZpYgh7uZWYIc7mZmCXK4m5klyOFuZpYgh7uZWYIc7mZmCXK4m5klyOFu\nZpYgh7uZWYIc7mZmCXK4m5klyOFuZpYgh7uZWYIc7mZmCXK4m5klyOFuZpYgh7uZWYJyhbukiyUt\nkbRU0g2HaPcJSSGpqXYlmplZV3Ua7pIagLuA6cBUYJakqVXaDQb+GHi61kWamVnX5Bm5TwOWRsSy\niNgDPATMqNLur4C/AdpqWJ+ZmR2GPOE+FlhV9rol2/YWSWcD4yPih4c6kKTZkpolNbe2tna5WDMz\nyydPuKvKtnhrp9QHuB24vrMDRcSciGiKiKZRo0blr9LMzLokT7i3AOPLXo8D1pS9HgycAfynpOXA\n+cBcX1Q1MytOnnBfAEyRdJKk/sBMYO7+nRGxJSIaI2JSREwCngIujYjmbqnYzMw61Wm4R0Q7cB0w\nH1gEPBwRL0u6WdKl3V2gmZl1Xd88jSJiHjCvYttNB2l70ZGXZWZmR8LfUDUzS5DD3cwsQQ53M7ME\nOdzNzBLkcDczS5DD3cwsQQ53M7MEOdzNzBLkcDczS5DD3cwsQQ53M7MEOdzNzBLkcDczS5DD3cws\nQQ53M7MEOdzNzBLkcDczS5DD3cwsQQ53M7MEOdzNzBLkcDczS5DD3cwsQQ53M7MEOdzNzBLkcDcz\nS5DD3cwsQQ53M7MEOdzNzBLkcDczS5DD3cwsQbnCXdLFkpZIWirphir7/0TSQkkvSHpE0sTal2pm\nZnl1Gu6SGoC7gOnAVGCWpKkVzZ4DmiLiXcD3gL+pdaFmZpZfnpH7NGBpRCyLiD3AQ8CM8gYR8VhE\n7MxePgWMq22ZZmbWFXnCfSywqux1S7btYK4Cflxth6TZkpolNbe2tuav0szMuiRPuKvKtqjaULoC\naAJuq7Y/IuZERFNENI0aNSp/lWZm1iV9c7RpAcaXvR4HrKlsJOnDwJeBD0TE7tqUZ2ZmhyPPyH0B\nMEXSSZL6AzOBueUNJJ0N3A1cGhFv1r5MMzPrik7DPSLageuA+cAi4OGIeFnSzZIuzZrdBhwHfFfS\nryTNPcjhzMysDvJMyxAR84B5FdtuKnv+4RrXZWZmR8DfUDUzS5DD3cwsQQ53M7MEOdzNzBLkcDcz\nS5DD3cwsQQ53M7MEOdzNzBLkcDczS5DD3cwsQQ53M7MEOdzNzBLkcDczS5DD3cwsQQ53M7MEOdzN\nzBLkcDczS5DD3cwsQQ53M7MEOdzNzBLkcDczS5DD3cwsQQ53M7MEOdzNzBLkcDczS5DD3cwsQQ53\nM7MEOdzNzBLkcDczS5DD3cwsQQ53M7ME5Qp3SRdLWiJpqaQbquwfIOmfs/1PS5pU60LNzCy/TsNd\nUgNwFzAdmArMkjS1otlVwKaImAzcDtxa60LNzCy/PCP3acDSiFgWEXuAh4AZFW1mAP+UPf8e8CFJ\nql2ZZmbWFXnCfSywqux1S7atapuIaAe2ACMrDyRptqRmSc2tra2HV7GZWS81oKGBS848kQkjBnX7\nufrmaFNtBB6H0YaImAPMAWhqajpgv5lZyoYO6sc3Lj+nLufKM3JvAcaXvR4HrDlYG0l9gaHAxloU\naGZmXZcn3BcAUySdJKk/MBOYW9FmLvDp7PkngEcjwiNzM7OCdDotExHtkq4D5gMNwD0R8bKkm4Hm\niJgL/CPwbUlLKY3YZ3Zn0WZmdmh55tyJiHnAvIptN5U9bwN+r7almZnZ4fI3VM3MEuRwNzNLkMPd\nzCxBDnczswSpqE8sSmoFVhRy8q5rBNYXXUQP4z45kPvkQO6T6o6kXyZGxKjOGhUW7r2JpOaIaCq6\njp7EfXIg98mB3CfV1aNfPC1jZpYgh7uZWYIc7vnMKbqAHsh9ciD3yYHcJ9V1e794zt3MLEEeuZuZ\nJcjhbmaWIId7mc4WAs/aXCZpoaSXJT1Q7xrrLcfi6LdL+lX2eEXS5iLqrKccfTJB0mOSnpP0gqRL\niqiznnL0yURJj2T98Z+SxhVRZz1JukfSm5JeOsh+Sboj67MXJL2npgVEhB+l6w4NwGvAyUB/4Hlg\nakWbKcBzwPDs9fFF1110n1S0/wKlW0IXXnvB/5/MAT6XPZ8KLC+67h7QJ98FPp09/yDw7aLrrkO/\nXAi8B3jpIPsvAX5MaSW784Gna3l+j9x/Lc9C4FcDd0XEJoCIeLPONdZbnj4pNwt4sC6VFSdPnwQw\nJHs+lANXLktNnj6ZCjySPX+syv7kRMTjHHpFuhnA/VHyFDBM0uhand/h/mt5FgI/FThV0s8kPSXp\n4rpVV4w8fQKU/uwGTgIerUNdRcrTJ18BrpDUQmkdhC/Up7TC5OmT54GPZ88/BgyWNLIOtfVkuX+/\nDofD/dfyLPLdl9LUzEWURqn/IGlYN9dVpFwLn2dmAt+LiH3dWE9PkKdPZgH3RcQ4Sn96f1tSyr9r\nefrki8AHJD0HfABYDbR3d2E9XFd+v7os10pMR4m8C4E/FRF7gdclLaEU9gvqU2Ld5emT/WYC13Z7\nRcXL0ydXARcDRMQvJA2kdKOoVKfxOu2TiFgD/HcASccBH4+ILXWrsGfqyu9Xl6U8muiqPAuB/wvw\nmwCSGilN0yyra5X1ladPkPQOYDjwizrXV4Q8fbIS+BCApNOBgUBrXausr077RFJj2V8vNwL31LnG\nnmgu8KnsUzPnA1siYm2tDu5wz0REO7B/IfBFwMORLQQu6dKs2Xxgg6SFlC4KfSkiNhRTcffL2SdQ\nmoZ4KLKPAKQsZ59cD1wt6XlKF5ivTLlvcvbJRcASSa8AJwC3FFJsHUl6kNKA5x2SWiRdJekaSddk\nTeZRGhwuBb4FfL6m50/4/zkzs6OWR+5mZglyuJuZJcjhbmaWIIe7mVmCHO5mZglyuJuZJcjhbmaW\noP8PBvx8bya8SuMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xf160940>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(precisions,recalls)\n",
    "plt.title('precision-recall')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### note:\n",
    "使用两组不同的超参数训练的两个模型的precision-recall曲线的位置，可以判断两组超参数对应模型的优劣"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ROC曲线：Receiver Operation Characteristic Curve"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TPR=recall=TP/TP+FN\n",
    "FPR=FP/TN+FP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "fprs,tprs,thresholds=roc_curve(y_test,decisoin_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(17L,)\n",
      "(17L,)\n",
      "(17L,)\n"
     ]
    }
   ],
   "source": [
    "print fprs.shape\n",
    "print tprs.shape\n",
    "print thresholds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAADtdJREFUeJzt3W2IpWd9x/Hvz2yjtE20dqcQ98Fd\n6QYcQzF2SCJCjRjKJuDuG2t3RawluNU2lhIppFiixDetpZUK2+pixSrEGH1hBllJqY1YxN1mJDG6\nG7ZMNzEZE5oxSeML0Rj674tzYg9nZ/bcM3tmzp5rvh8YuB+uuc//2nPmlyvX/XBSVUiS2vKSSRcg\nSRo/w12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUoG2TeuHt27fXnj17JvXykjSV\nvvOd7/yoqmZGtZtYuO/Zs4eFhYVJvbwkTaUkP+jSzmkZSWqQ4S5JDTLcJalBhrskNchwl6QGjQz3\nJJ9J8lSS76+yP0k+kWQxyUNJ3jD+MiVJa9Fl5P5ZYP959t8I7Ov/HAH+8cLLkiRdiJHXuVfVN5Ps\nOU+Tg8Dnqvd9fSeSvCLJFVX15Jhq3HR3nnyMex784aTLkNSo2Vddzoff9roNfY1xzLnvAB4fWF/q\nbztHkiNJFpIsLC8vj+GlN8Y9D/6Q00/+eNJlSNK6jeMO1aywbcVv3a6qY8AxgLm5uYv6m7lnr7ic\nL/7RGyddhiStyzhG7kvAroH1ncATYziuJGmdxhHu88C7+1fNXAc8N83z7ZLUgpHTMkm+AFwPbE+y\nBHwY+CWAqvokcBy4CVgEfgL84UYVuxFWOnl6+skfM3vF5ROqSJIuXJerZQ6P2F/An4ytok324snT\nwTCfveJyDr5+xXPCkjQVJvbI34uJJ08ltcbHD0hSgwx3SWrQlpmWWe2uU0+eSmrRlhm5r3bXqSdP\nJbVoy4zcwROnkraOLTNyl6StpOmR++A8u3PrkraSpkfug/Pszq1L2kqaHrmD8+yStqamR+6StFUZ\n7pLUoGbD/c6Tj3HykWcmXYYkTUSz4f7iVTKeRJW0FTUb7gDX7n0l77x296TLkKRN13S4S9JWZbhL\nUoOm+jr31Z70CN6RKmlrm+qR+2pPegTvSJW0tU31yB28A1WSVjLVI3dJ0sqmcuT+4ly78+qStLKp\nHLkPBrvz6pJ0rqkcuYNz7ZJ0PlM5cpcknZ/hLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhrUKdyT\n7E9yJslikttW2L87yX1JHkjyUJKbxl+qJKmrkeGe5BLgKHAjMAscTjI71Owvgbur6mrgEPAP4y5U\nktRdl5H7NcBiVZ2tqueBu4CDQ20KePEhLy8HnhhfiZKktery+IEdwOMD60vAtUNtPgL8S5IPAL8C\n3DCW6iRJ69Jl5J4VttXQ+mHgs1W1E7gJ+HySc46d5EiShSQLy8vLa69WktRJl3BfAnYNrO/k3GmX\nm4G7Aarq28DLgO3DB6qqY1U1V1VzMzMz66tYkjRSl3C/H9iXZG+SS+mdMJ0favMY8FaAJK+lF+4O\nzSVpQkaGe1W9ANwC3As8TO+qmFNJ7khyoN/sg8B7k3wX+ALwnqoanrqRJG2STs9zr6rjwPGhbbcP\nLJ8G3jTe0iRJ6+UdqpLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIa\nZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGG\nuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGdQr3JPuTnEmymOS2\nVdq8I8npJKeS3DneMiVJazEy3JNcAhwFbgRmgcNJZofa7AP+AnhTVb0O+LMNqBWAO08+xslHntmo\nw0tSE7qM3K8BFqvqbFU9D9wFHBxq817gaFU9C1BVT423zP93z4M/BODg63ds1EtI0tTrEu47gMcH\n1pf62wZdCVyZ5FtJTiTZv9KBkhxJspBkYXl5eX0VA9fufSXvvHb3un9fklrXJdyzwrYaWt8G7AOu\nBw4Dn07yinN+qepYVc1V1dzMzMxaa5UkddQl3JeAXQPrO4EnVmhzT1X9vKoeAc7QC3tJ0gR0Cff7\ngX1J9ia5FDgEzA+1+QrwFoAk2+lN05wdZ6GSpO5GhntVvQDcAtwLPAzcXVWnktyR5EC/2b3A00lO\nA/cBf15VT29U0ZKk89vWpVFVHQeOD227fWC5gFv7P5KkCfMOVUlqkOEuSQ0y3CWpQYa7JDXIcJek\nBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ\n4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnu\nktQgw12SGtQp3JPsT3ImyWKS287T7u1JKsnc+EqUJK3VyHBPcglwFLgRmAUOJ5ldod1lwJ8CJ8dd\npCRpbbqM3K8BFqvqbFU9D9wFHFyh3UeBjwE/HWN9kqR16BLuO4DHB9aX+tt+IcnVwK6q+uoYa5Mk\nrVOXcM8K2+oXO5OXAB8HPjjyQMmRJAtJFpaXl7tXKUlaky7hvgTsGljfCTwxsH4ZcBXwjSSPAtcB\n8yudVK2qY1U1V1VzMzMz669aknReXcL9fmBfkr1JLgUOAfMv7qyq56pqe1Xtqao9wAngQFUtbEjF\nkqSRRoZ7Vb0A3ALcCzwM3F1Vp5LckeTARhcoSVq7bV0aVdVx4PjQtttXaXv9hZclSboQ3qEqSQ0y\n3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNd\nkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWp\nQYa7JDXIcJekBhnuktQgw12SGmS4S1KDOoV7kv1JziRZTHLbCvtvTXI6yUNJvp7k1eMvVZLU1chw\nT3IJcBS4EZgFDieZHWr2ADBXVb8FfBn42LgLlSR112Xkfg2wWFVnq+p54C7g4GCDqrqvqn7SXz0B\n7BxvmZKktegS7juAxwfWl/rbVnMz8LULKUqSdGG2dWiTFbbVig2TdwFzwJtX2X8EOAKwe/fujiVK\nktaqy8h9Cdg1sL4TeGK4UZIbgA8BB6rqZysdqKqOVdVcVc3NzMysp15JUgddwv1+YF+SvUkuBQ4B\n84MNklwNfIpesD81/jIlSWsxMtyr6gXgFuBe4GHg7qo6leSOJAf6zf4G+FXgS0keTDK/yuEkSZug\ny5w7VXUcOD607faB5RvGXJck6QJ4h6okNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWp\nQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpk\nuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ3aNukC1mr2\nVZdPugRJuuh1Cvck+4G/By4BPl1VfzW0/6XA54DfBp4Gfr+qHh1vqT0fftvrNuKwktSUkdMySS4B\njgI3ArPA4SSzQ81uBp6tqt8EPg789bgLlSR112XO/RpgsarOVtXzwF3AwaE2B4F/7i9/GXhrkoyv\nTEnSWnQJ9x3A4wPrS/1tK7apqheA54BfH0eBkqS16xLuK43Aax1tSHIkyUKSheXl5S71SZLWoUu4\nLwG7BtZ3Ak+s1ibJNuDlwDPDB6qqY1U1V1VzMzMz66tYkjRSl3C/H9iXZG+SS4FDwPxQm3ngD/rL\nbwf+rarOGblLkjbHyEshq+qFJLcA99K7FPIzVXUqyR3AQlXNA/8EfD7JIr0R+6GNLFqSdH6drnOv\nquPA8aFttw8s/xT4vfGWJklar0xq9iTJMvCDdf76duBHYyxnGtjnrcE+bw0X0udXV9XIk5YTC/cL\nkWShquYmXcdmss9bg33eGjajzz44TJIaZLhLUoOmNdyPTbqACbDPW4N93ho2vM9TOecuSTq/aR25\nS5LO46IO9yT7k5xJspjkthX2vzTJF/v7TybZs/lVjleHPt+a5HSSh5J8PcmrJ1HnOI3q80C7tyep\nJFN/ZUWXPid5R/+9PpXkzs2ucdw6fLZ3J7kvyQP9z/dNk6hzXJJ8JslTSb6/yv4k+UT/3+OhJG8Y\nawFVdVH+0Lsb9r+A1wCXAt8FZofa/DHwyf7yIeCLk657E/r8FuCX+8vv3wp97re7DPgmcAKYm3Td\nm/A+7wMeAH6tv/4bk657E/p8DHh/f3kWeHTSdV9gn38HeAPw/VX23wR8jd6DF68DTo7z9S/mkftW\nfI78yD5X1X1V9ZP+6gl6D3KbZl3eZ4CPAh8DfrqZxW2QLn1+L3C0qp4FqKqnNrnGcevS5wJe/B7N\nl3PuAwqnSlV9kxUeoDjgIPC56jkBvCLJFeN6/Ys53Lfic+S79HnQzfT+yz/NRvY5ydXArqr66mYW\ntoG6vM9XAlcm+VaSE/2vupxmXfr8EeBdSZboPe7kA5tT2sSs9e99TS7mL8ge23Pkp0jn/iR5FzAH\nvHlDK9p45+1zkpfQ++rG92xWQZugy/u8jd7UzPX0/u/s35NcVVX/s8G1bZQufT4MfLaq/jbJG+k9\njPCqqvrfjS9vIjY0vy7mkfvYniM/Rbr0mSQ3AB8CDlTVzzapto0yqs+XAVcB30jyKL25yfkpP6na\n9bN9T1X9vKoeAc7QC/tp1aXPNwN3A1TVt4GX0XsGS6s6/b2v18Uc7lvxOfIj+9yfovgUvWCf9nlY\nGNHnqnquqrZX1Z6q2kPvPMOBqlqYTLlj0eWz/RV6J89Jsp3eNM3ZTa1yvLr0+THgrQBJXksv3Fv+\nyrZ54N39q2auA56rqifHdvRJn1Eecbb5JuA/6Z1l/1B/2x30/rih9+Z/CVgE/gN4zaRr3oQ+/yvw\n38CD/Z/5Sde80X0eavsNpvxqmY7vc4C/A04D3wMOTbrmTejzLPAtelfSPAj87qRrvsD+fgF4Evg5\nvVH6zcD7gPcNvMdH+/8e3xv359o7VCWpQRfztIwkaZ0Md0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnu\nktQgw12SGvR/SJtKlMCvUk8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xd3ffe48>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fprs,tprs)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 计算ROC曲线下的面积"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROC曲线面积为： 0.993257528141\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "auc=roc_auc_score(y_test,decisoin_scores)\n",
    "print u\"ROC曲线面积为：\",auc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### note:roc曲线对非常有偏的数据并不敏感，对于有偏的数据需要使用precision-recall曲线。\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# -------------------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 多分类问题的混淆矩阵"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X=digits.data\n",
    "y=digits.target\n",
    "X_train,X_test,y_train,y_test=train_test_split(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.957777777778\n"
     ]
    }
   ],
   "source": [
    "log_reg_multi_labels=LogisticRegression()\n",
    "log_reg_multi_labels.fit(X_train,y_train)\n",
    "print log_reg_multi_labels.score(X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[34  0  0  0  0  0  0  0  0  0]\n",
      " [ 0 49  0  0  1  0  0  0  1  0]\n",
      " [ 0  0 54  1  0  0  0  0  1  0]\n",
      " [ 0  0  0 45  0  2  0  0  0  0]\n",
      " [ 0  1  0  0 44  0  0  0  0  0]\n",
      " [ 0  2  0  0  0 39  0  0  0  0]\n",
      " [ 0  1  0  0  0  0 40  0  0  0]\n",
      " [ 0  0  0  0  0  0  0 44  0  0]\n",
      " [ 0  4  0  0  0  0  0  0 38  0]\n",
      " [ 0  0  0  1  0  1  0  1  2 44]]\n"
     ]
    }
   ],
   "source": [
    "y_predict=log_reg_multi_labels.predict(X_test)\n",
    "\n",
    "conf_mat=confusion_matrix(y_test,y_predict)\n",
    "print conf_mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.957777777778\n",
      "0.957777777778\n"
     ]
    }
   ],
   "source": [
    "### precision and recall \n",
    "print precision_score(y_test,y_predict,average='micro')\n",
    "print recall_score(y_test,y_predict,average='micro')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 针对预测错误可以绘制错误矩阵\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "conf_mat_sum=np.sum(conf_mat,axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "err_mat=conf_mat / conf_mat_sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 将对角线的数字填充为0\n",
    "np.fill_diagonal(err_mat,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.02222222,\n",
       "         0.        ,  0.        ,  0.        ,  0.02380952,  0.        ],\n",
       "       [ 0.        ,  0.        ,  0.        ,  0.0212766 ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.02380952,  0.        ],\n",
       "       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.04878049,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.01960784,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.03921569,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.01960784,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.07843137,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.        ,  0.        ,  0.        ,  0.        ],\n",
       "       [ 0.        ,  0.        ,  0.        ,  0.0212766 ,  0.        ,\n",
       "         0.02439024,  0.        ,  0.02272727,  0.04761905,  0.        ]])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "err_mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAACdJJREFUeJzt3c+LXfUdxvHnaUbRxIpCuzGRRqHY\nilAiQ1EDIsaBpP4ISBcKCs3CbFqNIoj2bxDRRRGG2GwMuogBq5TUgGbRTTAmgsaxRdTGaMR0URU3\nUXy6uFdQG3POlPu9Z24+7xcEZiYnhw/DvPM9995zv+MkAlDLj4YeAMD0ET5QEOEDBRE+UBDhAwUR\nPlDQYOHb3mz7H7bfsf3wUHP0ZftS26/YXrJ91PaOoWfqw/Yq20dsvzj0LH3Yvsj2Httvj7/X1w49\nUxfbD4x/Jt60/Yzt84aeqcsg4dteJelPkrZIulLSnbavHGKWZfhK0oNJfinpGkm/n4GZJWmHpKWh\nh1iGJyTtS/ILSb/SCp/d9lpJ90maT3KVpFWS7hh2qm5Drfi/lvROkneTnJL0rKStA83SS5ITSQ6P\nP/5cox/ItcNOdWa210m6WdLOoWfpw/aFkq6X9JQkJTmV5D/DTtXLnKTzbc9JWi3po4Hn6TRU+Gsl\nffCtz49rhUf0bbbXS9og6eCwk3R6XNJDkr4eepCeLpd0UtKu8cOTnbbXDD3UmST5UNKjko5JOiHp\n0yQvDTtVt6HC92m+NhP3Dtu+QNJzku5P8tnQ8/wQ27dI+iTJa0PPsgxzkq6W9GSSDZK+kLSin/+x\nfbFGV6uXSbpE0hrbdw07Vbehwj8u6dJvfb5OM3B5ZPscjaLfnWTv0PN02CjpNtvva/RQ6kbbTw87\nUqfjko4n+eZKao9G/xGsZDdJei/JySRfStor6bqBZ+o0VPivSvq57ctsn6vRkyF/GWiWXmxbo8ee\nS0keG3qeLkkeSbIuyXqNvr8vJ1nRK1GSjyV9YPuK8Zc2SXprwJH6OCbpGturxz8jm7TCn5CURpdW\nU5fkK9t/kPQ3jZ4F/XOSo0PMsgwbJd0t6Q3br4+/9sckfx1wprPRvZJ2jxeEdyVtG3ieM0py0PYe\nSYc1euXniKTFYafqZt6WC9TDnXtAQYQPFET4QEGEDxRE+EBBg4dve/vQMyzHrM0rMfM0zNq8g4cv\naaa+YZq9eSVmnoaZmnclhA9gyprcwGObu4KAgSQ53ZvgvoMVHyiI8IGCCB8oiPCBgggfKIjwgYJ6\nhT9re+ADOLPO1/HHe+D/U9KCRnuivSrpziQ/uCUSr+MDw5nU6/gztwc+gDPrE/5M74EP4H/12Wyz\n1x7443cnzdQbFYCq+oTfaw/8JIsa7y7KY3xgZetzqT9ze+ADOLPOFX9G98AHcAa8LRc4y/C2XACn\nRfhAQYQPFET4QEGEDxQ0yK/JxmRs3ry5yXn37dvX5Lyz6NZbb2127hdeeKHZubuw4gMFET5QEOED\nBRE+UBDhAwURPlAQ4QMFET5QEOEDBRE+UBDhAwURPlAQ4QMFET5QEOEDBRE+UBDhAwURPlAQ4QMF\nET5QEOEDBRE+UBDbaze2sLDQ7Nxsg43/Fys+UBDhAwURPlAQ4QMFET5QEOEDBRE+UFBn+LYvtf2K\n7SXbR23vmMZgANrpcwPPV5IeTHLY9o8lvWZ7f5K3Gs8GoJHOFT/JiSSHxx9/LmlJ0trWgwFoZ1mP\n8W2vl7RB0sEWwwCYjt736tu+QNJzku5P8tlp/n67pO0TnA1AI73Ct32ORtHvTrL3dMckWZS0OD4+\nE5sQwMT1eVbfkp6StJTksfYjAWitz2P8jZLulnSj7dfHf37TeC4ADXVe6if5uyRPYRYAU8Kde0BB\nhA8URPhAQYQPFET4QEFOJn+vDTfwzLZdu3Y1Oe+2bduanBfflaTzVThWfKAgwgcKInygIMIHCiJ8\noCDCBwoifKAgwgcKInygIMIHCiJ8oCDCBwoifKAgwgcKInygIMIHCiJ8oCDCBwoifKAgwgcKInyg\nIMIHCpqp7bVvuOGGFqeVJB04cKDZuYFpYnttAKdF+EBBhA8URPhAQYQPFET4QEGEDxTUO3zbq2wf\nsf1iy4EAtLecFX+HpKVWgwCYnl7h214n6WZJO9uOA2Aa+q74j0t6SNLXDWcBMCWd4du+RdInSV7r\nOG677UO2D01sOgBN9FnxN0q6zfb7kp6VdKPtp79/UJLFJPNJ5ic8I4AJ6ww/ySNJ1iVZL+kOSS8n\nuav5ZACa4XV8oKC55Ryc5ICkA00mATA1rPhAQYQPFET4QEGEDxRE+EBBy3pWf2jssgtMBis+UBDh\nAwURPlAQ4QMFET5QEOEDBRE+UBDhAwURPlAQ4QMFET5QEOEDBRE+UBDhAwURPlAQ4QMFET5QEOED\nBRE+UBDhAwURPlDQTO2yy064wGSw4gMFET5QEOEDBRE+UBDhAwURPlAQ4QMF9Qrf9kW299h+2/aS\n7WtbDwagnb438DwhaV+S39o+V9LqhjMBaKwzfNsXSrpe0u8kKckpSafajgWgpT6X+pdLOilpl+0j\ntnfaXtN4LgAN9Ql/TtLVkp5MskHSF5Ie/v5BtrfbPmT70IRnBDBhfcI/Lul4koPjz/do9B/BdyRZ\nTDKfZH6SAwKYvM7wk3ws6QPbV4y/tEnSW02nAtBU32f175W0e/yM/ruStrUbCUBrvcJP8rokLuGB\nswR37gEFET5QEOEDBRE+UBDhAwURPlCQk0z+pPbkTwqglyTuOoYVHyiI8IGCCB8oiPCBgggfKIjw\ngYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGCCB8oqO9v\ny10RWuwI/A27c2NS4KzBig8URPhAQYQPFET4QEGEDxRE+EBBhA8U1Ct82w/YPmr7TdvP2D6v9WAA\n2ukM3/ZaSfdJmk9ylaRVku5oPRiAdvpe6s9JOt/2nKTVkj5qNxKA1jrDT/KhpEclHZN0QtKnSV5q\nPRiAdvpc6l8saaukyyRdImmN7btOc9x224dsH5r8mAAmqc+l/k2S3ktyMsmXkvZKuu77ByVZTDKf\nZH7SQwKYrD7hH5N0je3VHr2FbZOkpbZjAWipz2P8g5L2SDos6Y3xv1lsPBeAhtziPe62m7xxnvfj\nA92SdP4wc+ceUBDhAwURPlAQ4QMFET5QEOEDBc3U9tqz+JLbwsJCs3Pv37+/yXm3bt3a5LzPP/98\nk/NK0pYtW5qc9/bbb29yXkm65557mp27Cys+UBDhAwURPlAQ4QMFET5QEOEDBRE+UBDhAwURPlAQ\n4QMFET5QEOEDBRE+UBDhAwURPlAQ4QMFET5QEOEDBRE+UBDhAwURPlBQq1+aeVLSv3oe/hNJ/574\nEO3M2rwSM0/DSpn3Z0l+2nVQk/CXw/ahJPODDrEMszavxMzTMGvzcqkPFET4QEErIfzFoQdYplmb\nV2LmaZipeQd/jA9g+lbCig9gyggfKIjwgYIIHyiI8IGC/gs43DXR+Dk6qQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xfe763c8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(err_mat,cmap=plt.cm.gray)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}