数据分析实践/多分类的评估问题.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  8.79681649e-01   1.20307538e-01   1.08131372e-05]\n",
      " [  7.99706325e-01   2.00263292e-01   3.03825365e-05]\n",
      " [  8.53796795e-01   1.46177302e-01   2.59031285e-05]\n",
      " [  8.25383127e-01   1.74558937e-01   5.79356669e-05]\n",
      " [  8.97323628e-01   1.02665167e-01   1.12050036e-05]\n",
      " [  9.26986574e-01   7.30004562e-02   1.29693872e-05]\n",
      " [  8.95064974e-01   1.04895775e-01   3.92506205e-05]\n",
      " [  8.61839691e-01   1.38141399e-01   1.89095833e-05]\n",
      " [  8.03156719e-01   1.96758495e-01   8.47861140e-05]\n",
      " [  7.95421554e-01   2.04552763e-01   2.56832240e-05]\n",
      " [  8.92083069e-01   1.07910759e-01   6.17176870e-06]\n",
      " [  8.63364991e-01   1.36600589e-01   3.44201798e-05]\n",
      " [  7.88177618e-01   2.11794929e-01   2.74526810e-05]\n",
      " [  8.35079702e-01   1.64888155e-01   3.21426418e-05]\n",
      " [  9.28349898e-01   7.16491356e-02   9.66254924e-07]\n",
      " [  9.64535656e-01   3.54620850e-02   2.25877936e-06]\n",
      " [  9.40906153e-01   5.90890027e-02   4.84421830e-06]\n",
      " [  8.91740161e-01   1.08245661e-01   1.41772124e-05]\n",
      " [  8.96525617e-01   1.03467608e-01   6.77567332e-06]\n",
      " [  9.23615524e-01   7.63726510e-02   1.18248373e-05]\n",
      " [  8.30668332e-01   1.69316458e-01   1.52093733e-05]\n",
      " [  9.21914602e-01   7.80675598e-02   1.78384021e-05]\n",
      " [  9.26584671e-01   7.34068679e-02   8.46162713e-06]\n",
      " [  8.67785629e-01   1.32146178e-01   6.81931916e-05]\n",
      " [  8.41271506e-01   1.58655904e-01   7.25903122e-05]\n",
      " [  7.77263282e-01   2.22695181e-01   4.15365716e-05]\n",
      " [  8.81389224e-01   1.18568969e-01   4.18075826e-05]\n",
      " [  8.69974782e-01   1.30013638e-01   1.15794893e-05]\n",
      " [  8.60034106e-01   1.39955486e-01   1.04082979e-05]\n",
      " [  8.32052869e-01   1.67892968e-01   5.41625519e-05]\n",
      " [  8.07811588e-01   1.92136477e-01   5.19350231e-05]\n",
      " [  8.72544939e-01   1.27438925e-01   1.61360155e-05]\n",
      " [  9.33948477e-01   6.60477336e-02   3.78900866e-06]\n",
      " [  9.46250501e-01   5.37475145e-02   1.98493064e-06]\n",
      " [  7.95421554e-01   2.04552763e-01   2.56832240e-05]\n",
      " [  8.47610513e-01   1.52377535e-01   1.19520539e-05]\n",
      " [  8.70019435e-01   1.29976367e-01   4.19728170e-06]\n",
      " [  7.95421554e-01   2.04552763e-01   2.56832240e-05]\n",
      " [  8.31024910e-01   1.68917216e-01   5.78737851e-05]\n",
      " [  8.57737250e-01   1.42246900e-01   1.58501104e-05]\n",
      " [  9.00222082e-01   9.97646975e-02   1.32206853e-05]\n",
      " [  6.90741687e-01   3.09094698e-01   1.63615590e-04]\n",
      " [  8.66068303e-01   1.33887708e-01   4.39884356e-05]\n",
      " [  9.16308833e-01   8.36288777e-02   6.22895883e-05]\n",
      " [  9.15519114e-01   8.44392129e-02   4.16734713e-05]\n",
      " [  8.20309627e-01   1.79642381e-01   4.79919885e-05]\n",
      " [  9.09855663e-01   9.01327650e-02   1.15724381e-05]\n",
      " [  8.51214451e-01   1.48746052e-01   3.94971199e-05]\n",
      " [  8.95519736e-01   1.04472911e-01   7.35323849e-06]\n",
      " [  8.51563342e-01   1.48419676e-01   1.69821772e-05]\n",
      " [  2.98900777e-02   8.60393138e-01   1.09716785e-01]\n",
      " [  3.74487166e-02   7.05572459e-01   2.56978825e-01]\n",
      " [  1.17957675e-02   7.48252356e-01   2.39951876e-01]\n",
      " [  1.32920493e-02   6.51770445e-01   3.34937506e-01]\n",
      " [  1.09868088e-02   6.98832091e-01   2.90181101e-01]\n",
      " [  1.07669519e-02   5.83013186e-01   4.06219862e-01]\n",
      " [  2.15200540e-02   5.37732882e-01   4.40747064e-01]\n",
      " [  1.08418544e-01   7.68766189e-01   1.22815267e-01]\n",
      " [  1.77270021e-02   8.27562690e-01   1.54710308e-01]\n",
      " [  3.30493839e-02   5.28708770e-01   4.38241846e-01]\n",
      " [  2.93117962e-02   7.72717609e-01   1.97970595e-01]\n",
      " [  4.09569813e-02   6.19765980e-01   3.39277039e-01]\n",
      " [  1.95378252e-02   8.79697992e-01   1.00764183e-01]\n",
      " [  8.73285529e-03   5.96503817e-01   3.94763328e-01]\n",
      " [  1.67434866e-01   7.12756209e-01   1.19808925e-01]\n",
      " [  4.75535678e-02   8.43626581e-01   1.08819852e-01]\n",
      " [  1.22530319e-02   4.23869480e-01   5.63877488e-01]\n",
      " [  3.84753639e-02   8.50175432e-01   1.11349204e-01]\n",
      " [  3.09968794e-03   5.96264678e-01   4.00635634e-01]\n",
      " [  3.59781700e-02   8.08752206e-01   1.55269624e-01]\n",
      " [  6.20745751e-03   2.73106189e-01   7.20686354e-01]\n",
      " [  5.81151228e-02   8.19701311e-01   1.22183566e-01]\n",
      " [  1.95840574e-03   5.33800891e-01   4.64240703e-01]\n",
      " [  8.77628703e-03   7.04654010e-01   2.86569703e-01]\n",
      " [  3.69274341e-02   8.38990091e-01   1.24082475e-01]\n",
      " [  3.61807169e-02   8.28744840e-01   1.35074443e-01]\n",
      " [  8.14489700e-03   7.77156946e-01   2.14698157e-01]\n",
      " [  4.64006697e-03   5.23164549e-01   4.72195384e-01]\n",
      " [  1.33500103e-02   5.63205976e-01   4.23444014e-01]\n",
      " [  1.28473017e-01   8.31361691e-01   4.01652917e-02]\n",
      " [  3.60902230e-02   8.03217466e-01   1.60692311e-01]\n",
      " [  5.05096042e-02   8.46149445e-01   1.03340951e-01]\n",
      " [  5.69724571e-02   8.11250984e-01   1.31776559e-01]\n",
      " [  1.22453086e-03   3.99201919e-01   5.99573550e-01]\n",
      " [  1.03123407e-02   3.65034695e-01   6.24652965e-01]\n",
      " [  4.17476538e-02   4.77844283e-01   4.80408063e-01]\n",
      " [  1.90525287e-02   7.45629538e-01   2.35317933e-01]\n",
      " [  7.05352060e-03   7.56932682e-01   2.36013798e-01]\n",
      " [  5.57541864e-02   6.67410837e-01   2.76834977e-01]\n",
      " [  2.10790319e-02   6.62362244e-01   3.16558724e-01]\n",
      " [  8.98003281e-03   5.99716389e-01   3.91303578e-01]\n",
      " [  1.52196906e-02   6.32329159e-01   3.52451150e-01]\n",
      " [  3.47695685e-02   7.98625645e-01   1.66604786e-01]\n",
      " [  9.15416570e-02   7.95877151e-01   1.12581192e-01]\n",
      " [  1.98418694e-02   6.40871800e-01   3.39286330e-01]\n",
      " [  4.81040905e-02   7.31039981e-01   2.20855929e-01]\n",
      " [  3.44565240e-02   6.77463657e-01   2.88079819e-01]\n",
      " [  3.38822929e-02   7.96899915e-01   1.69217792e-01]\n",
      " [  2.54574647e-01   6.90791330e-01   5.46340233e-02]\n",
      " [  3.63488963e-02   7.04234211e-01   2.59416893e-01]\n",
      " [  1.86036022e-04   1.48760823e-01   8.51053141e-01]\n",
      " [  8.09069371e-04   2.94422745e-01   7.04768186e-01]\n",
      " [  2.78126551e-04   3.30535386e-01   6.69186488e-01]\n",
      " [  4.56288643e-04   3.38732197e-01   6.60811514e-01]\n",
      " [  2.51393977e-04   2.57092194e-01   7.42656412e-01]\n",
      " [  6.03186905e-05   3.82744333e-01   6.17195349e-01]\n",
      " [  2.04838186e-03   2.81103453e-01   7.16848165e-01]\n",
      " [  1.23247784e-04   4.24393655e-01   5.75483097e-01]\n",
      " [  1.59929758e-04   4.23195996e-01   5.76644074e-01]\n",
      " [  3.56390886e-04   1.52542892e-01   8.47100717e-01]\n",
      " [  2.99635433e-03   2.78024684e-01   7.18978962e-01]\n",
      " [  6.45242833e-04   3.55681241e-01   6.43673516e-01]\n",
      " [  6.81029987e-04   2.98859721e-01   7.00459249e-01]\n",
      " [  6.28418142e-04   2.96807692e-01   7.02563890e-01]\n",
      " [  6.10997845e-04   1.74593604e-01   8.24795398e-01]\n",
      " [  1.09757190e-03   1.73257823e-01   8.25644605e-01]\n",
      " [  7.99254871e-04   3.48929847e-01   6.50270898e-01]\n",
      " [  1.93443479e-04   2.38473708e-01   7.61332849e-01]\n",
      " [  1.30064976e-05   4.20137191e-01   5.79849802e-01]\n",
      " [  6.81548718e-04   4.69975854e-01   5.29342597e-01]\n",
      " [  5.04477452e-04   2.25292722e-01   7.74202801e-01]\n",
      " [  1.33913767e-03   2.30143290e-01   7.68517573e-01]\n",
      " [  3.82097113e-05   4.28006955e-01   5.71954836e-01]\n",
      " [  2.05299242e-03   4.00421888e-01   5.97525119e-01]\n",
      " [  6.77847072e-04   2.37204010e-01   7.62118143e-01]\n",
      " [  4.56383243e-04   3.97527741e-01   6.02015876e-01]\n",
      " [  3.19858866e-03   3.83866887e-01   6.12934525e-01]\n",
      " [  3.42364119e-03   3.27541103e-01   6.69035256e-01]\n",
      " [  3.00544917e-04   2.98288662e-01   7.01410793e-01]\n",
      " [  6.78376797e-04   5.10705151e-01   4.88616472e-01]\n",
      " [  1.61719140e-04   4.27941843e-01   5.71896438e-01]\n",
      " [  6.44775841e-04   3.44845359e-01   6.54509865e-01]\n",
      " [  2.75279882e-04   2.78027400e-01   7.21697320e-01]\n",
      " [  2.07731418e-03   4.90652652e-01   5.07270034e-01]\n",
      " [  3.54683506e-04   4.42580814e-01   5.57064503e-01]\n",
      " [  1.82017584e-04   3.42008155e-01   6.57809828e-01]\n",
      " [  6.30908753e-04   1.28602511e-01   8.70766580e-01]\n",
      " [  9.21940559e-04   3.20888055e-01   6.78190005e-01]\n",
      " [  4.29311663e-03   3.18426266e-01   6.77280618e-01]\n",
      " [  1.16680587e-03   3.00989509e-01   6.97843685e-01]\n",
      " [  4.46290865e-04   2.02461924e-01   7.97091785e-01]\n",
      " [  2.15227432e-03   2.48822456e-01   7.49025270e-01]\n",
      " [  8.09069371e-04   2.94422745e-01   7.04768186e-01]\n",
      " [  2.91162367e-04   2.24919706e-01   7.74789132e-01]\n",
      " [  4.50477099e-04   1.53984748e-01   8.45564775e-01]\n",
      " [  1.15724730e-03   2.33616548e-01   7.65226205e-01]\n",
      " [  9.19025197e-04   3.79220387e-01   6.19860588e-01]\n",
      " [  1.45811816e-03   2.98379693e-01   7.00162189e-01]\n",
      " [  1.09779827e-03   1.31785617e-01   8.67116585e-01]\n",
      " [  1.68397530e-03   2.81057800e-01   7.17258224e-01]] [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 2\n",
      " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n",
      " 2 2]\n"
     ]
    }
   ],
   "source": [
    "from sklearn import datasets\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import label_binarize\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import confusion_matrix, precision_score, accuracy_score,recall_score, f1_score,roc_auc_score\n",
    "\n",
    "\n",
    "iris = datasets.load_iris()\n",
    "x, y = iris.data, iris.target\n",
    "n_class = len(set(iris.target))\n",
    "y_one_hot = label_binarize(y, np.arange(n_class))\n",
    "\n",
    "# alpha = np.logspace(-2, 2, 20)  #设置超参数范围\n",
    "# model = LogisticRegressionCV(Cs = alpha, cv = 3, penalty = 'l2')  #使用L2正则化\n",
    "model = LogisticRegression()\n",
    "model.fit(x, y)\n",
    "y_score = model.predict(x)\n",
    "y_score_pro = model.predict_proba(x) #返回的是\n",
    "y_score_one_hot = label_binarize(y_score, np.arange(n_class))\n",
    "print(y_score_pro,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion_matrix\n",
      " [[50  0  0]\n",
      " [ 0 45  5]\n",
      " [ 0  1 49]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix, precision_score, accuracy_score,recall_score, f1_score,roc_auc_score\n",
    "\n",
    "obj1 = confusion_matrix(y, y_score)\n",
    "print('confusion_matrix\\n', obj1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 2\n",
      " 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2\n",
      " 2 2]\n",
      "accuracy:0.96\n",
      "precision:0.96\n",
      "recall:0.96\n",
      "f1-score:0.96\n",
      "AUC:0.97\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(y)\n",
    "print('accuracy:{}'.format(accuracy_score(y, y_score)))\n",
    "print('precision:{}'.format(precision_score(y, y_score,average='micro')))\n",
    "print('recall:{}'.format(recall_score(y, y_score,average='micro')))\n",
    "print('f1-score:{}'.format(f1_score(y, y_score,average='micro')))\n",
    "print('AUC:{}\\n'.format(roc_auc_score(y_one_hot, y_score_one_hot,average='micro')))\n",
    " #形式一：原始值（0或1或2）\n",
    "     #形式二：各类概率值\n",
    "         # 形式三：one-hot值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy:0.96\n",
      "\n",
      "precision:0.96\n",
      "\n",
      "recall:0.96\n",
      "\n",
      "f1-score:0.96\n",
      "\n",
      "AUC:0.97\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('accuracy:{}\\n'.format(accuracy_score(y_one_hot, y_score_one_hot)))\n",
    "print('precision:{}\\n'.format(precision_score(y_one_hot, y_score_one_hot,average='micro')))\n",
    "print('recall:{}\\n'.format(recall_score(y_one_hot, y_score_one_hot,average='micro')))\n",
    "print('f1-score:{}\\n'.format(f1_score(y_one_hot, y_score_one_hot,average='micro')))\n",
    "print('AUC:{}\\n'.format(roc_auc_score(y_one_hot, y_score_one_hot,average='micro')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XucTfX+x/HXJ0KoFLpRhyLHMDPuJDmkRBe6KIp0c0tK\n/Ajdy6mjc3ROSop0v6Akow6lFCndlOuM1JDQjejizpjP74+97TNpzGzMnrX3zPv5eOzHmb322mu9\n1zitz3zX5bPM3REREQE4JOgAIiISP1QUREQkQkVBREQiVBRERCRCRUFERCJUFEREJEJFQUREIlQU\nREQkQkVBREQiSgYdYH9VqlTJq1WrFnQMEZGE8vnnn//s7pXzmy/hikK1atWYP39+0DFERBKKmX0b\nzXw6fCQiIhEqCiIiEqGiICIiESoKIiISoaIgIiIRKgoiIhKhoiAiIhEqCiIiEqGiICIiESoKIiIS\noaIgIiIRKgoiIhKhoiAiIhEqCiIiEhGzomBmT5nZOjNbuo/PzcweNrNMM1tsZg1ilUVERKITy5HC\nM0C7PD5vD9QMv3oBj8Uwi4iIRCFmRcHd3wc25jFLR+A5D/kYqGBmx8cqj4iI5C/IJ69VAdbkeL82\nPO2HYOLEl2ue/pT3lq8POoaIFJCs337Cs3ZxaMWqrBpxXtBx9ikhHsdpZr0IHWLipJNOKtR1a+cs\nIgfDs3ez6Yv/8uv7z1Hq+FM57vL7g46UpyCLwnfAiTneVw1P+xN3HweMA2jUqJHHPtr/BFkQWteq\nzNPXNAls/SJycJYtW0aPHj2YN28e7dq1Y+zYsYX+h+3+CrIoTAP6mdlEoCnwm7sHdugovxFBPA/3\nRCT+bN26lZYtW5Kdnc1zzz1Ht27dMLOgY+UrZkXBzCYArYBKZrYWuAs4FMDdHwemA+cCmcBW4JpY\nZdnjQA8Fta5VOQZpRKQo+vLLL6lVqxZly5blxRdfJDU1lWOPPTboWFGLWVFw98vz+dyBGwpqfQVx\n7F+Ha0TkQG3bto27776bkSNH8uyzz9KtWzfatm0bdKz9lhAnmqMRbUHQjl9ECtr7779Pjx49+Prr\nr+nRowfnn39+0JEOWJEoCtc8/WnkZx37F5HCdM8993D33XdTvXp13nnnHdq0aRN0pIOS8L2Pch42\n0rF/ESksoSPg0KhRIwYMGMCSJUsSviAA2J4NSxSNGjXy+fPnA38uCDosJCKx9vPPPzNgwABq1qzJ\nnXfeGXScqJnZ5+7eKL/5EnqkoIIgIoXF3Xn55ZdJSkpi4sSJHHJIQu8+96lInFNQQRCRWPr+++/p\n27cvaWlpNGrUiHfeeYeUlJSgY8VE0Sx1IiIF6Mcff+Tdd9/lX//6Fx999FGRLQhQREYKIiIFbeXK\nlUybNo2bb76ZBg0asHr1aipUqBB0rJjTSEFEJIfdu3fzn//8h7p163LXXXfx448/AhSLggAqCiIi\nEenp6Zx++ukMHDiQM888k/T0dI477rigYxUqHT4SESHUwO5vf/sbZsZLL71Ely5dEqKBXUFTURCR\nYi0jI4PatWtTtmxZJk6cSGpqKpUrF98bYXX4SESKpa1btzJ48GCSk5N54YUXADjrrLOKdUGABB4p\n5Ox3JCKyP2bPnk3Pnj3JzMykd+/edOjQIehIcSNhRwrqdyQiB+Kuu+6idevWuDvvvvsujz/+OEce\neWTQseJGwhaFPXQ3s4hEY0+ftyZNmvB///d/LF68mNatWwecKv4kfFEQEcnL+vXrueKKK7j33nsB\nOO+88xg5ciRly5YNOFl8UlEQkSLJ3XnppZeoXbs2kydPplSpUkFHSggqCiJS5Kxdu5YOHTrQtWtX\natSowYIFCxg2bFjQsRKCioKIFDnr16/n/fff59///jcffvghderUCTpSwkjYS1JFRHLKzMzk9ddf\nZ8CAAdSvX581a9ZwxBFHBB0r4WikICIJLSsri5EjR5KcnMw999zDTz/9BKCCcIBUFEQkYS1ZsoTm\nzZszePBg2rZtS3p6Oscee2zQsRKaDh+JSELaunUrrVu35pBDDmHixIlcdtllxbKBXUFLyKKgFhci\nxdfSpUupU6cOZcuWZdKkSaSmplKpUqWgYxUZCXn4SC0uRIqfLVu2MHDgQFJSUiIN7Nq0aaOCUMAS\ncqSwh1pciBQPs2bNomfPnnzzzTf07duXjh07Bh2pyErIkYKIFB933HEHZ511FiVLlmTOnDk8+uij\nurIohlQURCQuZWdnA9C8eXNuueUWFi1aRMuWLQNOVfSpKIhIXFm3bh1dunThnnvuAaB9+/Y88MAD\nHHbYYQEnKx5UFEQkLrg7L7zwArVr1+a1115TF9OAxLQomFk7M1tuZplmNjSXz480s9fNbJGZpZvZ\nNbHMIyLxac2aNZx//vlceeWV1KpViwULFjBkyJCgYxVLMSsKZlYCeBRoDyQBl5tZ0l6z3QBkuHsq\n0Ap40Mzy7G+76uctMUgrIkHasGEDH374IaNGjWLu3LkkJe29q5DCEstLUpsAme6+EsDMJgIdgYwc\n8zhwuIVuQywPbASy8lroph1ZlEf3KIgkuq+++opp06YxaNAg6tWrx5o1azj88MODjlXsxfLwURVg\nTY73a8PTchoN1Aa+B5YA/d09O5qF6x4FkcSUlZXFAw88QEpKCvfdd1+kgZ0KQnwI+kTzOcBC4ASg\nHjDazP50AbKZ9TKz+WY2v7ADikjBWbRoEU2bNmXo0KGce+65ZGRkqIFdnIllUfgOODHH+6rhaTld\nA0zxkEzgG+Cvey/I3ce5eyN3bxSztCISU1u3bqVNmzZ89913TJ48mSlTpnD88ccHHUv2Esui8BlQ\n08yqh08edwGm7TXPaqANgJkdC9QCVsYwk4gUssWLF+PulC1blldeeYWMjAwuueSSoGPJPsSsKLh7\nFtAPeAtYBrzs7ulm1sfM+oRnGw40N7MlwCxgiLv/HKtMIlJ4Nm/eTP/+/alXrx7PP/88AK1bt+bo\no48OOJnkJaYN8dx9OjB9r2mP5/j5e6BtLDOISOF7++236dWrF6tWraJfv35cdNFFQUeSKEU1UjCz\nUmZWI9ZhRCTx3XbbbbRt25bSpUszd+5cHnnkEV1ZlEDyLQpmdh6hy0XfDr+vZ2avxTqYiCSWPQ3s\nWrRowbBhw1i4cCEtWrQIOJXsr2hGCvcCTYFfAdx9IaBRg4gA8OOPP9KpUyfuvvtuINTA7v7776dM\nmTLBBpMDEk1R2OXuv+41zWMRRkQSh7vzzDPPkJSUxBtvvKFnHBQR0ZxoXmZmlwGHmFl14Cbg49jG\nEpF49u2339KrVy9mzpxJixYtGD9+PLVq1Qo6lhSAaEYK/YCGQDYwBdgB9I9lKBGJb7/++iufffYZ\no0ePZs6cOSoIRUg0I4Vz3H0IEOlja2YXEyoQIlJMLF++nGnTpjF48GBSU1NZvXo15cuXDzqWFLBo\nRgq35zLttoIOIiLxadeuXfzjH/8gNTWVESNGsG7dOgAVhCJqnyMFMzsHaAdUMbN/5/joCEKHkkSk\niFuwYAHXXXcdCxYsoFOnTowePZpjjjkm6FgSQ3kdPloHLAW2A+k5pm8C/vQUNREpWrZu3crZZ5/N\noYceyquvvsrFF18cdCQpBPssCu6+AFhgZi+6+/ZCzCQiAVqwYAH16tWjbNmyTJ48mdTUVI466qig\nY0khieacQhUzm2hmi83sqz2vmCcTkUK1adMm+vXrR4MGDSIN7Fq1aqWCUMxEUxSeAZ4GjNDzll8G\nJsUwk4gUsjfffJO6desyZswY+vfvr0NFxVg0RaGsu78F4O4r3P12QsVBRIqAYcOG0b59e8qVK8eH\nH37IQw89pCuLirFo7lPYYWaHACvCz0H4DlDLQ5EEt3v3bkqUKEGrVq0oWbIkt99+O6VLlw46lgQs\nmqIwAChHqL3FfcCRwLWxDCUisfPDDz9www03UKdOHYYPH84555zDOeecE3QsiRP5Hj5y90/cfZO7\nr3b3K929A7Aq9tFEpCC5O08//TRJSUnMmDFDJ5AlV3kWBTNrbGYXmlml8Ps6ZvYc8EmhpBORArFq\n1Sratm3LtddeS3JyMosWLWLgwIFBx5I4tM+iYGb/AF4EugJvmtndwHvAIuDUQkknIgXit99+44sv\nvmDMmDHMnj2bU0/Vf8KSu7zOKXQEUt19m5kdDawBkt19ZeFEE5GDkZGRwbRp0xg6dGikgV25cuWC\njiVxLq/DR9vdfRuAu28EvlJBEIl/O3fu5O9//zv169dn5MiRkQZ2KggSjbxGCieb2Z722AZUz/Ee\nd9fdLSJxZv78+Vx33XUsXryYLl26MGrUKDWwk/2SV1G4ZK/3o2MZREQOzpYtWzjnnHMoU6YMaWlp\ndOjQIehIkoDyaog3qzCDiMiB+eKLL6hXrx7lypXjtddeIyUlhQoVKgQdSxJUNG0uRCQO/f777/Tt\n25eGDRvywgsvANCyZUsVBDko0dzRLCJxZvr06fTu3Zvvv/+egQMHcsklex/tFTkwUY8UzExNUUTi\nwJAhQzjvvPM44ogjmDdvHg8++KCuLJICk+9IwcyaAE8S6nl0kpmlAj3c/cZYhxOREHcnOzubEiVK\n0KZNG8qUKcOtt96qBnZS4KIZKTwMnA9sAHD3RUDrWIYSkf/57rvvuPDCC7nrrrsAaNu2Lffcc48K\ngsRENEXhEHf/dq9pu2MRRkT+x9154oknSEpKYubMmVSqVCnoSFIMRHOieU34EJKbWQngRkCP4xSJ\noW+++YbrrruO9957j1atWvHEE09Qo0aNoGNJMRDNSOF6YCBwEvAT0Cw8LV9m1s7MlptZppkN3cc8\nrcxsoZmlm9mcaIOLFGWbN29m8eLFjB07llmzZqkgSKGJZqSQ5e5d9nfB4VHFo8DZwFrgMzOb5u4Z\nOeapAIwB2rn7ajPT/fhSbC1dupRp06Zx6623kpyczOrVqylbtmzQsaSYiWak8JmZTTezq8xsfx7D\n2QTIdPeV7r4TmEio82pOVwBT3H01gLuv24/lixQJO3fu5J577qFBgwb85z//iTSwU0GQIETz5LVT\ngL8DDYElZjbVzKIZOVQh1G57j7XhaTmdChxlZrPN7HMz657bgsysl5nNN7P5UaxXJGF89tlnNGzY\nkLvvvptLL72UjIwMNbCTQEV185q7z3P3m4AGwO+EHr5TEEoSKjbnAecAd5jZn57+4e7j3L2Ruzcq\noPWKBG7Lli20a9eOX375hWnTpvHiiy9SuXLloGNJMRfNzWvlCR326QLUBtKA5lEs+zvgxBzvq4an\n5bQW2ODuW4AtZvY+kEo+Vze1rqX/cCRxzZ8/nwYNGlCuXDnS0tJITk7myCOPDDqWCBDdSGEpoSuO\n/unuNdz9/9w9mmc0fwbUNLPqZlaKUFGZttc8aUALMytpZmWBpsCy/Bb89DVNoli9SHz57bff6N27\nN40bN440sGvRooUKgsSVaK4+Otnds/d3we6eZWb9gLeAEsBT7p5uZn3Cnz/u7svM7E1gMZANjHf3\npfu7LpF49/rrr9OnTx9+/PFHBg0aRKdOnYKOJJIrc/fcPzB70N3/z8xeA/40U1BPXit9fE3f8cPX\nQaxa5IAMHjyYkSNHkpyczJNPPknjxo2DjiTFkJl9Hs152bxGCpPC/6snronsJ3dn9+7dlCxZkrZt\n23LEEUcwZMgQSpUqFXQ0kTztc6QQmcGsn7uPzm9aYdFIQeLd2rVruf7660lJSeG+++4LOo4IEP1I\nIZoTzdfmMu26/Y8kUrRlZ2czduxYkpKSePfddznuuOOCjiSy3/Z5+MjMOhO6Yqi6mU3J8dHhwK+x\nDiaSSFauXMm1117LnDlzaNOmDePGjePkk08OOpbIfsvrnMKnhJ6hUJVQD6M9NgELYhlKJNFs2bKF\njIwMxo8fz7XXXouZBR1J5IDke04h3uicgsSLJUuWkJaWxu233w7Atm3bOOywwwJOJZK7gz6nsKeN\ntZn9YmYbc7x+MbONBRlWJJHs2LGDO++8kwYNGvDwww9HGtipIEhRkNeJ5j2P3KwEVM7x2vNepNj5\n+OOPadCgAcOHD+fyyy9n2bJlamAnRco+zynkuIv5ROB7d99pZi2AFOAFQo3xRIqNLVu2cN5551Gu\nXDmmT59O+/btg44kUuCiuSR1KqFHcZ4CPA3UBF6KaSqROPLJJ5+QnZ1NuXLleP3110lPT1dBkCIr\nmqKQ7e67gIuBR9x9AH9+LoJIkfPrr7/So0cPmjVrFmlg17x5cw4/fH+eNSWSWKJ6HKeZXQpcCVwY\nnnZo7CKJBG/q1Kn07duXdevWMWTIEC699NKgI4kUimjvaG5NqHX2SjOrDkyIbSyR4AwcOJCLLrqI\nY445hk8++YQRI0boyiIpNvIdKbj7UjO7CahhZn8l9NxlNXSRIiVnA7tzzz2XihUrcsstt3DooRoU\nS/ESTUO8M4DnCT01zYDjgCvd/cPYx/sz3bwmBW316tX06dOH+vXrq4GdFFkF2RDvP8C57n66uzcn\n9DzlUQcbUCRo2dnZjBkzhjp16jBnzhxOOOGEoCOJBC6aE82l3D1jz5vw09LUFF4SWmZmJtdeey1z\n587l7LPPZty4cVSrVi3oWCKBi6YofGFmjxO6YQ2gK2qIJwlu+/btfPXVVzz99NNcddVVamAnEhbN\nOYUywE1Ai/CkuYTuV9ge42y50jkFOVALFy4kLS2Nu+66CwgVhjJlygScSqRwRHtOIc+iYGbJwClA\nurvHxZ5YRUH21/bt2xk+fDgPPPAAlSpVYvHixepXJMVOQXRJvZVQi4uuwNtmltsT2ETi2rx586hf\nvz73338/3bp1IyMjQwVBJA95nVPoCqS4+xYzqwxMB54qnFgiB2/Lli1ccMEFlC9fnjfffJNzzjkn\n6EgicS+vorDD3bcAuPt6M4vm8lWRwH300Uc0bdqUcuXK8cYbb1C3bl31KxKJUl47+pPNbEr49Rpw\nSo73U/L4nkggfvnlF6699lqaN2/O888/D8Bpp52mgiCyH/IaKVyy1/vRsQwicjCmTJnCDTfcwPr1\n6xk2bBidO3cOOpJIQsrrITuzCjOIyIEaMGAADz30EPXq1WP69OnUr18/6EgiCSuam9dE4k7OBnbn\nn38+xxxzDIMGDVIDO5GDlO/Na/FG9ynIqlWr6N27Nw0aNOAf//hH0HFEEkJBNsTbs8DSBxdJ5OBk\nZ2fzyCOPULduXebNm8df/vKXoCOJFDn5FgUza2JmS4Cvw+9TzeyRmCcTyeHrr7+mZcuW3HTTTZxx\nxhksXbqUPn36BB1LpMiJZqTwMHA+sAHA3RcRehJbvsysnZktN7NMMxuax3yNzSzLzDpFs1wpfnbu\n3MmKFSt47rnnmD59ukYJIjESTVE4xN2/3Wva7vy+ZGYlgEeB9kAScLmZJe1jvgeAmVFkkWJkwYIF\n3H333QDUqVOHVatWceWVV6qjqUgMRVMU1phZE8DNrISZ3Qx8FcX3mhB6dOdKd98JTAQ65jLfjcCr\nwLpoQ0vRtn37doYNG0bjxo0ZO3Ys69evB6B0aZ3WEom1aIrC9cBA4CTgJ6BZeFp+qgBrcrxfG54W\nYWZVgIuAx6IJK0XfBx98QGpqKiNGjKB79+5kZGRQuXLloGOJFBv53qfg7uuALjFa/0PAEHfPzuuQ\ngJn1AnoBlDquRoyiSNA2b95Mx44dOeKII5g5cyZnn3120JFEip18i4KZPQH86WYGd++Vz1e/A07M\n8b5qeFpOjYCJ4YJQCTjXzLLcfepe6xoHjIPQfQr5ZZbE8sEHH9C8eXPKly/Pf//7X+rWrUv58uWD\njiVSLEVz+OgdYFb49SFwDLAjiu99BtQ0s+rhZzp3AablnMHdq7t7NXevBkwG+u5dEKTo2rBhA927\nd+eMM86INLBr1qyZCoJIgKI5fDQp53szex74IIrvZZlZP+AtoATwlLunm1mf8OePH1hkSXTuzuTJ\nk+nXrx8bN27kjjvuoEuXWB2hFJH9cSC9j6oDx0Yzo7tPJ/RwnpzTci0G7n71AWSRBDRgwABGjRpF\nw4YNmTlzJqmpqUFHEpGwaM4p/ML/zikcAmwE9nkjmkhu3J2srCwOPfRQOnTowAknnMDAgQMpWVI9\nGUXiSZ4N8Sx0BvhE/neCONsD7qCnhniJ55tvvqFXr140bNiQESNGBB1HpFgqkIZ44QIw3d13h1+6\n8keitnv3bkaNGkXdunX55JNPOPnkk4OOJCL5iGbsvtDM6rv7gpinkSLjq6++4uqrr+ajjz6iffv2\njB07lhNPPDH/L4pIoPZZFMyspLtnAfWBz8xsBbAFMEKDiAaFlFESUFZWFt9++y0vvPACV1xxhfoV\niSSIvEYKnwINgA6FlEUS3Pz580lLS2P48OEkJSWxcuVK9SsSSTB5nVMwAHdfkdurkPJJAti2bRu3\n3HILTZs25amnnlIDO5EEltdIobKZDdzXh+7+7xjkkQQzZ84cevToQWZmJj179uSf//wnFSpUCDqW\niBygvIpCCaA84RGDyN42b97MxRdfTIUKFZg1axZnnnlm0JFE5CDlVRR+cPd7Cy2JJIy5c+dy+umn\nU758eWbMmEGdOnUoV65c0LFEpADke05BZI+ff/6Zbt260bJly0gDuyZNmqggiBQheY0U2hRaColr\n7s7LL7/MjTfeyC+//MJdd92lBnYiRdQ+i4K7byzMIBK/+vfvzyOPPELjxo2ZNWsWycnJQUcSkRhR\nNzLJlbuza9cuSpUqxUUXXcRf/vIXbr75ZkqUKBF0NBGJoTwb4sUjNcSLvRUrVtCzZ08aNWrEP//5\nz6DjiEgBKJCGeFK87N69m3//+98kJyfz+eefU6tWraAjiUgh0+EjAeDLL7/kqquu4tNPP+WCCy7g\nscceo0qVKkHHEpFCpqIgAGRnZ/P9998zYcIEOnfurAZ2IsWUikIx9umnn5KWlsZ9991HUlISK1as\noFSpUkHHEpEA6ZxCMbR161YGDRrEaaedxrPPPhtpYKeCICIqCsXMe++9R3JyMg8++CA9e/YkPT2d\nypUrBx1LROKEDh8VI5s3b+bSSy+lQoUKvPfee7Rq1SroSCISZzRSKAZmz55NdnZ2pIHd4sWLVRBE\nJFcqCkXY+vXrufzyy2ndujUvvPACAI0bN6Zs2bIBJxOReKXDR0WQuzNhwgRuuukmNm3axPDhw9XA\nTkSioqJQBN144408+uijNGvWjCeffJKkpKSgI4lIglBRKCKys7PJysqiVKlSdOrUiRo1anDjjTeq\ngZ2I7Bc1xCsCvv76a3r27Enjxo3517/+FXQcEYlDaohXDGRlZTFy5EhSUlJYuHAhtWvXDjqSiCQ4\nHT5KUMuWLaN79+7Mnz+fjh07MmbMGE444YSgY4lIglNRSGA//fQTkyZN4tJLL1UDOxEpEDE9fGRm\n7cxsuZllmtnQXD7vamaLzWyJmc0zs9RY5kl0H3/8McOGDQOgdu3arFixgssuu0wFQUQKTMyKgpmV\nAB4F2gNJwOVmtve1kd8Af3P3ZGA4MC5WeRLZli1bGDBgAM2bN+fFF1+MNLA79NBDA04mIkVNLEcK\nTYBMd1/p7juBiUDHnDO4+zx3/yX89mOgagzzJKR33nmHunXr8tBDD9G3b181sBORmIrlOYUqwJoc\n79cCTfOY/zpgRm4fmFkvoBdAqeNqFFS+uLd582a6dOnC0Ucfzfvvv88ZZ5wRdCQRKeLi4pJUM2tN\nqCgMye1zdx/n7o2iuca2KHj33XfZvXs35cuX56233mLRokUqCCJSKGJZFL4DTszxvmp42h+YWQow\nHujo7htimCfu/fTTT1x22WW0adMm0sCuYcOGHHbYYQEnE5HiIpZF4TOgpplVN7NSQBdgWs4ZzOwk\nYApwpbt/FcMscc3def7550lKSoo8HvOKK64IOpaIFEMxO6fg7llm1g94CygBPOXu6WbWJ/z548Cd\nQEVgTPiyyqzicogopxtuuIHHHnuM0047jSeffFJ3JotIYNT7KCDZ2dns2rWL0qVLM2fOHBYvXkzf\nvn3VwE5EYiLa3kcqCgFYvnw5PXr0oGnTpowcOTLoOCJSDKghXhzatWsXI0aMIDU1laVLl5KcnBx0\nJBGRP1Dvo0KSnp7OlVdeyYIFC7j44ot59NFHOe6444KOJSLyByoKhaREiRJs3LiRyZMnc8kllwQd\nR0QkVzp8FEPz5s1jyJDQ/Xh//etfyczMVEEQkbimohADmzdv5qabbqJFixZMmjSJn3/+GYCSJTUw\nE5H4pqJQwGbOnEndunUZPXo0/fr1Y+nSpVSqVCnoWCIiUdGfrgVo8+bNdO3alYoVKzJ37lxOP/30\noCOJiOwXjRQKwNtvvx1pYDdz5kwWLlyogiAiCUlF4SD88MMPXHLJJbRt25YXX3wRgPr161OmTJmA\nk4mIHBgVhQPg7jzzzDMkJSXx3//+lxEjRqiBnYgUCTqncACuv/56xo4dS4sWLRg/fjy1atUKOpJI\nXNq1axdr165l+/btQUcpNsqUKUPVqlUP+HG9KgpRytnA7oorriAlJYU+ffpwyCEabInsy9q1azn8\n8MOpVq0a4U7IEkPuzoYNG1i7di3Vq1c/oGVojxaFZcuWccYZZ3DrrbcC0LJlS/r27auCIJKP7du3\nU7FiRRWEQmJmVKxY8aBGZtqr5WHXrl3cf//91KtXjy+//JL69esHHUkk4aggFK6D/X2rKOxDeno6\nTZo04bbbbqNjx45kZGTQrVu3oGOJyAGYOnUqZsaXX34JwOzZszn//PP/MM/VV1/N5MmTgdAfhEOH\nDqVmzZo0aNCA0047jRkzZkS1rh07dtC5c2dq1KhB06ZNWbVqVa7zTZo0iZSUFOrUqRNphwPw7bff\n0qZNG1JSUmjVqhVr166NfFaiRAnq1atHvXr16NChw/78CqKmorAPJUuW5LfffmPKlCm8/PLLHHvs\nsUFHEpEDNGHCBFq0aMGECROimv+OO+7ghx9+YOnSpXzxxRdMnTqVTZs2RfXdJ598kqOOOorMzEwG\nDBjwhx3+Hhs2bGDw4MHMmjWL9PR0fvzxR2bNmgXAoEGD6N69O4sXL+bOO+9k2LBhke8ddthhLFy4\nkIULFzLOlts2AAANDElEQVRt2rQ/LbcgqCjkMHfuXAYNGgRArVq1+Oqrr7jooosCTiUiB2Pz5s18\n8MEHPPnkk0ycODHf+bdu3coTTzzBI488QunSpQE49thjueyyy6JaX1paGldddRUAnTp1YtasWez9\nMLOVK1dSs2ZNKleuDMBZZ53Fq6++CkBGRgZnnnkmAK1btyYtLS26DS0guvoI2LRpE0OHDmXMmDFU\nr16doUOHUqlSJTWwEylA1Yb+NybLXTXivDw/T0tLo127dpx66qlUrFiRzz//PM/5MzMzOemkkzji\niCNy/bxz584sX778T9MHDhxI9+7d+e677zjxxBOB0BGHI488kg0bNvyhB1qNGjVYvnw5q1atomrV\nqkydOpWdO3cCkJqaypQpU+jfvz+vvfYamzZtYsOGDZETyA0aNKBUqVIMHTqUCy+8MM9tORDFfq83\nY8YMevfuzdq1a7n55pv5+9//Trly5YKOJSIFZMKECfTv3x+ALl26MGHCBC644IJc543mJO2kSZMO\nOtNRRx3FY489RufOnTnkkENo3rw5K1asAGDkyJH069ePZ555hpYtW1KlSpXIs9u//fZbqlSpwsqV\nKznzzDNJTk7mlFNOOeg8ORXrorBp0ya6d+/OMcccw7x582jWrFnQkUSKrPz+oo+FjRs38u6777Jk\nyRLMjN27d2NmXHXVVfzyyy9/mrdSpUrUqFGD1atX8/vvv+c6WshvpFClShXWrFlD1apVycrK4rff\nfqNixYp/mv+CCy6IFKdx48ZFdvwnnHACU6ZMAUKHvl599VUqVKgAQJUqVQA4+eSTadWqFQsWLCjw\nooC7J9Sr1HE1/GBkZ2f7jBkzPCsry93dFy5c6Nu3bz+oZYpI7jIyMgJd/9ixY71Xr15/mNayZUuf\nPXu2V6tWLZJv1apVftJJJ/mvv/7q7u6DBw/2q6++2nfs2OHu7uvWrfOXX345qnWOHj3ae/fu7e7u\nEyZM8EsvvTTX+X766Sd3d9+4caOnpqb68uXL3d19/fr1vnv3bnd3v/XWW/2OO+6IzLdnX7V+/Xqv\nUaOGp6en57rs3H7vwHyPYh8b+E5+f18HUxS+//57v/DCCx3wZ5999oCXIyLRCbootGrVymfMmPGH\naaNGjfI+ffr4Bx984E2bNvXU1FRv1KiRz5w5MzLPjh07fPDgwX7KKad4nTp1vEmTJv7mm29Gtc5t\n27Z5p06d/JRTTvHGjRv7ihUrIp+lpqZGfu7SpYvXrl3ba9eu7RMmTIhMf+WVV7xGjRpes2ZNv+66\n6yKF4MMPP/S6det6SkqK161b18ePH7/PDAdTFMz3Oise70ofX9N3/PD1fn3H3Xn66acZOHAgO3bs\n4N5772XAgAE6kSwSY8uWLaN27dpBxyh2cvu9m9nn7t4ov+8Wi71inz59GDduHC1btmT8+PHUrFkz\n6EgiInGpyBaF3bt3s2vXLsqUKUO3bt2oX78+vXr1Ur8iEZE8FMk9ZHp6Oqeffnqkgd0ZZ5yhjqYi\nIlEoUnvJnTt3Mnz4cOrXr09mZiaNGzcOOpJIsZdo5y0T3cH+vovM4aMlS5bQtWtXlixZQpcuXXj4\n4Ycjt5CLSDDKlCkTuRtX3VJjz8PPUziYRwIXmaJQqlQptm7dSlpaWsy6B4rI/qlatSpr165l/fr1\nQUcpNvY8ee1AxfSSVDNrB4wCSgDj3X3EXp9b+PNzga3A1e7+RV7LzHlJ6pw5c5g2bRoPPvggEDq5\nvOeuQBER+Z9oL0mN2TkFMysBPAq0B5KAy80saa/Z2gM1w69ewGPRLPv333/n+uuvp1WrVkydOpWf\nf/4ZQAVBROQgxfJEcxMg091XuvtOYCLQca95OgLPhW+4+xioYGbH57XQ7B1bqFOnDuPGjWPgwIEs\nWbLkD90HRUTkwMXynEIVYE2O92uBplHMUwX4YZ9L3bSeI0+oxeTJk2nadO/FiYjIwUiIE81m1ovQ\n4SWAHenp6UuLWUfTSsDPQYcoZNrm4kHbXHj+Es1MsSwK3wEn5nhfNTxtf+fB3ccB4wDMbH40J0uK\nEm1z8aBtLh7ifZtjeU7hM6CmmVU3s1JAF2Dvh4pOA7pbSDPgN3ff96EjERGJqZiNFNw9y8z6AW8R\nuiT1KXdPN7M+4c8fB6YTuhw1k9AlqdfEKo+IiOQvpucU3H06oR1/zmmP5/jZgRv2c7HjCiBaotE2\nFw/a5uIhrrc54Z6nICIisVOkGuKJiMjBiduiYGbtzGy5mWWa2dBcPjczezj8+WIzaxBEzoIUxTZ3\nDW/rEjObZ2apQeQsSPltc475GptZlpl1Ksx8BS2a7TWzVma20MzSzWxOYWcsaFH8//pIM3vdzBaF\ntznhzy2a2VNmts7Mlu7j8/jdf0XzzM7CfhE6Mb0COBkoBSwCkvaa51xgBmBAM+CToHMXwjY3B44K\n/9y+OGxzjvneJXR+qlPQuWP8b1wByABOCr8/JujchbDNtwIPhH+uDGwESgWd/SC3uyXQAFi6j8/j\ndv8VryOFmLTIiHP5brO7z3P3X8JvPyZ0X0cii+bfGeBG4FVgXWGGi4FotvcKYIq7rwZw9+KwzQ4c\nHm6QWZ5QUcgq3JgFy93fJ7Qd+xK3+694LQr7an+xv/Mkkv3dnusI/aWRyPLdZjOrAlxElM0S41w0\n/8anAkeZ2Wwz+9zMuhdautiIZptHA7WB74ElQH93zy6ceIGJ2/1XQrS5kD8ys9aEikKLoLMUgoeA\nIe6eXUwe0lISaAi0AQ4DPjKzj939q2BjxdQ5wELgTOAU4G0zm+vuvwcbq3iK16JQYC0yEkhU22Nm\nKcB4oL27byikbLESzTY3AiaGC0Il4Fwzy3L3qYUTsUBFs71rgQ3uvgXYYmbvA6lAohaFaLb5GmCE\nhw62Z5rZN8BfgU8LJ2Ig4nb/Fa+Hj4pji4x8t9nMTgKmAFcWkb8c891md6/u7tXcvRowGeiboAUB\novv/dRrQwsxKmllZQp2FlxVyzoIUzTavJjQywsyOBWoBKws1ZeGL2/1XXI4UvBi2yIhym+8EKgJj\nwn85Z3kcN9bKT5TbXGREs73uvszM3gQWA9mEnliY62WNiSDKf+PhwDNmtoTQ1ThD3D2hO6ea2QSg\nFVDJzNYCdwGHQvzvv3RHs4iIRMTr4SMREQmAioKIiESoKIiISISKgoiIRKgoiIhIhIqCxB0z2x3u\nErrnVS2PeavtqxPlfq5zdriT5yIz+9DMah3AMi40s6Qc7+81s7Py+c50M6twgDk/M7N6UXzn5vA9\nDyL5UlGQeLTN3evleK0qpPV2dfdU4FngXwfw/QuBSFFw9zvd/Z28vuDu57r7r/u5nj05xxBdzpsB\nFQWJioqCJITwiGCumX0RfjXPZZ46ZvZpeHSx2Mxqhqd3yzF9rJmVyGd17wM1wt9tY2YLLPQMi6fM\nrHR4+ggzywivZ2Q4TwfgX+H1nGJmz5hZJws9T+CVHDlbmdkb4Z9XmVmlA8z5ETmaqJnZY2Y230LP\nJLgnPO0m4ATgPTN7LzytrZl9FP49vmJm5fNZjxQjKgoSjw7LcejotfC0dcDZ7t4A6Aw8nMv3+gCj\n3L0eoZ5Ja82sdnj+08PTdwNd81n/BcASMysDPAN0dvdkQh0ArjezioQ6t9Zx9xTg7+4+j1DrgsHh\n0c2KHMt7B2hqZuXC7zsTaiEdcYA52wE5W37cFr7DPQX4m5mluPvDhLqPtnb31uECdDtwVvh3OR8Y\nmM96pBiJyzYXUuxtC+8YczoUGB0+hr6bUIvpvX0E3GZmVQk9k+BrM2tDqOvoZ+HWIIex7+cyvGhm\n24BVhJ7hUAv4JkefqWeBGwi1et4OPBn+i/+NvDYm3OrhTeACM5sMnAfcstds+5uzFKFnD+T8PV1m\nZr0I/Xd9PKFDWYv3+m6z8PQPw+spRej3JgKoKEjiGAD8RKhj6CGEdsp/4O4vmdknhHa6082sN6Fe\nOs+6+7Ao1tHV3efveWNmR+c2U3gn34TQjrwT0I9Q2+e8TAzPtxGY7+6b9vp8v3ICnxM6n/AIcLGZ\nVQcGAY3d/RczewYok8t3DXjb3S+PYj1SDOnwkSSKI4Efwg9fuZJQc7U/MLOTgZXhQyZphA6jzAI6\nmdkx4XmONrO/RLnO5UA1M6sRfn8lMCd8DP5Id59OqFjteVb2JuDwfSxrDqHHM/Zkr0NHYfuVM9xm\n+g6gmZn9FTgC2AL8ZqFOo+1zzJ4z18fA6Xu2yczKmVluoy4pplQUJFGMAa4ys0WEeu1vyWWey4Cl\nZrYQqEvocYcZhI6hzzSzxcDbhA6t5MvdtxPqXvlKuINnNvA4oR3sG+HlfcD/jslPBAaHT0yfstey\ndhM6zNSeXA43HUhOd98GPEjoPMYiYAHwJfAS8GGOWccBb5rZe+6+HrgamBBez0eEfp8igLqkiohI\nDhopiIhIhIqCiIhEqCiIiEiEioKIiESoKIiISISKgoiIRKgoiIhIhIqCiIhE/D+sEC/LFFLCXQAA\nAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x15bfaa50cc0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import roc_curve\n",
    "%matplotlib inline\n",
    "auc = roc_auc_score(y_one_hot, y_score_pro,average='micro')\n",
    "fpr, tpr, thresholds = roc_curve(y_one_hot.ravel(),y_score_pro.ravel())  # ravel()表示平铺开来\n",
    "plt.plot(fpr, tpr, linewidth = 2,label='AUC=%.3f' % auc)\n",
    "plt.plot([0,1],[0,1], 'k--')\n",
    "plt.axis([0,1.1,0,1.1])\n",
    "plt.xlabel('False Postivie Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.legend()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}