Model Evaluation and hyperparamter tuning in python.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases'\n",
    "                 '/breast-cancer-wisconsin/wdbc.data', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "      <th>25</th>\n",
       "      <th>26</th>\n",
       "      <th>27</th>\n",
       "      <th>28</th>\n",
       "      <th>29</th>\n",
       "      <th>30</th>\n",
       "      <th>31</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>842302</td>\n",
       "      <td>M</td>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>...</td>\n",
       "      <td>25.38</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>842517</td>\n",
       "      <td>M</td>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>...</td>\n",
       "      <td>24.99</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>84300903</td>\n",
       "      <td>M</td>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>...</td>\n",
       "      <td>23.57</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>84348301</td>\n",
       "      <td>M</td>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>...</td>\n",
       "      <td>14.91</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>84358402</td>\n",
       "      <td>M</td>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>...</td>\n",
       "      <td>22.54</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0  1      2      3       4       5        6        7       8   \\\n",
       "0    842302  M  17.99  10.38  122.80  1001.0  0.11840  0.27760  0.3001   \n",
       "1    842517  M  20.57  17.77  132.90  1326.0  0.08474  0.07864  0.0869   \n",
       "2  84300903  M  19.69  21.25  130.00  1203.0  0.10960  0.15990  0.1974   \n",
       "3  84348301  M  11.42  20.38   77.58   386.1  0.14250  0.28390  0.2414   \n",
       "4  84358402  M  20.29  14.34  135.10  1297.0  0.10030  0.13280  0.1980   \n",
       "\n",
       "        9    ...        22     23      24      25      26      27      28  \\\n",
       "0  0.14710   ...     25.38  17.33  184.60  2019.0  0.1622  0.6656  0.7119   \n",
       "1  0.07017   ...     24.99  23.41  158.80  1956.0  0.1238  0.1866  0.2416   \n",
       "2  0.12790   ...     23.57  25.53  152.50  1709.0  0.1444  0.4245  0.4504   \n",
       "3  0.10520   ...     14.91  26.50   98.87   567.7  0.2098  0.8663  0.6869   \n",
       "4  0.10430   ...     22.54  16.67  152.20  1575.0  0.1374  0.2050  0.4000   \n",
       "\n",
       "       29      30       31  \n",
       "0  0.2654  0.4601  0.11890  \n",
       "1  0.1860  0.2750  0.08902  \n",
       "2  0.2430  0.3613  0.08758  \n",
       "3  0.2575  0.6638  0.17300  \n",
       "4  0.1625  0.2364  0.07678  \n",
       "\n",
       "[5 rows x 32 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0], dtype=int64)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "X = df.loc[:, 2:].values\n",
    "y = df.loc[:, 1].values\n",
    "le = LabelEncoder()\n",
    "y = le.fit_transform(y)\n",
    "le.transform(['M', 'B'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = \\\n",
    "    train_test_split(X, y, test_size=0.20, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.947\n"
     ]
    }
   ],
   "source": [
    "#Combining transformers and estimators in a pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "pipe_lr = Pipeline([('scl', StandardScaler()),\n",
    "                    ('pca', PCA(n_components=2)),\n",
    "                    ('clf', LogisticRegression(random_state=1))])\n",
    "\n",
    "pipe_lr.fit(X_train, y_train)\n",
    "print('Test Accuracy: %.3f' % pipe_lr.score(X_test, y_test))\n",
    "y_pred = pipe_lr.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold: 1, Class dist.: [256 153], Acc: 0.891\n",
      "Fold: 2, Class dist.: [256 153], Acc: 0.978\n",
      "Fold: 3, Class dist.: [256 153], Acc: 0.978\n",
      "Fold: 4, Class dist.: [256 153], Acc: 0.913\n",
      "Fold: 5, Class dist.: [256 153], Acc: 0.935\n",
      "Fold: 6, Class dist.: [257 153], Acc: 0.978\n",
      "Fold: 7, Class dist.: [257 153], Acc: 0.933\n",
      "Fold: 8, Class dist.: [257 153], Acc: 0.956\n",
      "Fold: 9, Class dist.: [257 153], Acc: 0.978\n",
      "Fold: 10, Class dist.: [257 153], Acc: 0.956\n"
     ]
    }
   ],
   "source": [
    "#running cross validation\n",
    "import numpy as np\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "\n",
    "kfold = StratifiedKFold(y=y_train, n_folds=10, random_state=1)\n",
    "\n",
    "scores = []\n",
    "for k , (train, test) in enumerate(kfold):\n",
    "    pipe_lr.fit(X_train[train],y_train[train])\n",
    "    score = pipe_lr.score(X_train[test],y_train[test])\n",
    "    scores.append(score)\n",
    "    print('Fold: %s, Class dist.: %s, Acc: %.3f' % (k+1,np.bincount(y_train[train]), score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV accuracy scores: [ 0.89130435  0.97826087  0.97826087  0.91304348  0.93478261  0.97777778\n",
      "  0.93333333  0.95555556  0.97777778  0.95555556]\n",
      "CV accuracy: 0.950 +/- 0.029\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import cross_val_score\n",
    "\n",
    "scores = cross_val_score(estimator=pipe_lr,X=X_train,y=y_train,cv=10,n_jobs=1)\n",
    "print('CV accuracy scores: %s' % scores)\n",
    "print('CV accuracy: %.3f +/- %.3f' % (np.mean(scores), np.std(scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.learning_curve import learning_curve\n",
    "\n",
    "pipe_lr = Pipeline([('scl', StandardScaler()),('clf', LogisticRegression(penalty='l2', random_state=0))])\n",
    "\n",
    "train_sizes, train_scores, test_scores =\\\n",
    "                learning_curve(estimator=pipe_lr,X=X_train,y=y_train,train_sizes=np.linspace(0.1, 1.0, 10),cv=10,n_jobs=1)\n",
    "\n",
    "train_mean = np.mean(train_scores, axis=1)\n",
    "train_std = np.std(train_scores, axis=1)\n",
    "test_mean = np.mean(test_scores, axis=1)\n",
    "test_std = np.std(test_scores, axis=1)\n",
    "\n",
    "plt.plot(train_sizes, train_mean,color='blue', marker='o',markersize=5, label='training accuracy')\n",
    "\n",
    "plt.fill_between(train_sizes,train_mean + train_std,train_mean - train_std,alpha=0.15, color='blue')\n",
    "\n",
    "plt.plot(train_sizes, test_mean,color='green', linestyle='--',marker='s', markersize=5,label='validation accuracy')\n",
    "\n",
    "plt.fill_between(train_sizes,test_mean + test_std,test_mean - test_std,alpha=0.15, color='green')\n",
    "\n",
    "plt.grid()\n",
    "plt.xlabel('Number of training samples')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend(loc='lower right')\n",
    "plt.ylim([0.8, 1.0])\n",
    "plt.tight_layout()\n",
    "# plt.savefig('./figures/learning_curve.png', dpi=300)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Addressing over- and underfitting with validation curves\n",
    "\"\"\"\n",
    "We can see that a linear function (polynomial with degree 1) is not sufficient to fit the training samples. This is \n",
    "called underfitting. A polynomial of degree 4 approximates the true function almost perfectly. However, for higher \n",
    "degrees the model will overfit the training data, i.e. it learns the noise of the training data. We evaluate quantitatively \n",
    "overfitting / underfitting by using cross-validation. We calculate the mean squared error (MSE) on the validation set, \n",
    "the higher, the less likely the model generalizes correctly from the training data.\n",
    "\n",
    "To validate a model we need a scoring function for example accuracy for classifiers. The proper way of choosing \n",
    "multiple hyperparameters of an estimator are of course grid search or similar methods that select the hyperparameter \n",
    "with the maximum score on a validation set or multiple validation sets. Note that if we optimized the hyperparameters\n",
    "based on a validation score the validation score is biased and not a good estimate of the generalization any longer. \n",
    "To get a proper estimate of the generalization we have to compute the score on another test set.\n",
    "\n",
    "However, it is sometimes helpful to plot the influence of a single hyperparameter on the training score and the validation \n",
    "score to find out whether the estimator is overfitting or underfitting for some hyperparameter values.\n",
    "If the training score and the validation score are both low, the estimator will be underfitting. If the training score\n",
    "is high and the validation score is low, the estimator is overfitting and otherwise it is working very well. A low training \n",
    "score and a high validation score is usually not possible\n",
    "\n",
    "A learning curve shows the validation and training score of an estimator for varying numbers of training samples. \n",
    "It is a tool to find out how much we benefit from adding more training data and whether the estimator suffers more \n",
    "from a variance error or a bias error. If both the validation score and the training score converge to a value that \n",
    "is too low with increasing size of the training set, we will not benefit much from more training data. In the following\n",
    "plot you can see an example: naive Bayes roughly converges to a low score\n",
    "\n",
    "http://scikit-learn.org/stable/modules/learning_curve.html#validation-curve\n",
    "\"\"\"\n",
    "\n",
    "from sklearn.learning_curve import validation_curve\n",
    "\n",
    "param_range = [0.001, 0.01, 0.1, 1.0, 10.0, 100.0]\n",
    "train_scores, test_scores = validation_curve(estimator=pipe_lr, X=X_train, y=y_train, param_name='clf__C', \n",
    "                                             param_range=param_range,cv=10)\n",
    "\n",
    "train_mean = np.mean(train_scores, axis=1)\n",
    "train_std = np.std(train_scores, axis=1)\n",
    "test_mean = np.mean(test_scores, axis=1)\n",
    "test_std = np.std(test_scores, axis=1)\n",
    "\n",
    "plt.plot(param_range, train_mean, color='blue', marker='o', markersize=5, label='training accuracy')\n",
    "\n",
    "plt.fill_between(param_range, train_mean + train_std,train_mean - train_std, alpha=0.15,color='blue')\n",
    "\n",
    "plt.plot(param_range, test_mean, color='green', linestyle='--', marker='s', markersize=5, label='validation accuracy')\n",
    "\n",
    "plt.fill_between(param_range, test_mean + test_std,test_mean - test_std, alpha=0.15, color='green')\n",
    "\n",
    "plt.grid()\n",
    "plt.xscale('log')\n",
    "plt.legend(loc='lower right')\n",
    "plt.xlabel('Parameter C')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.ylim([0.8, 1.0])\n",
    "plt.tight_layout()\n",
    "# plt.savefig('./figures/validation_curve.png', dpi=300)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.978021978022\n",
      "{'clf__C': 0.1, 'clf__kernel': 'linear'}\n",
      "Test accuracy: 0.965\n"
     ]
    }
   ],
   "source": [
    "#Tuning hyperparameters via grid search to select the hyperparameter with the maximum score on a validation set or multiple validation sets\n",
    "\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "pipe_svc = Pipeline([('scl', StandardScaler()),('clf', SVC(random_state=1))])\n",
    "\n",
    "param_range = [0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0, 1000.0]\n",
    "\n",
    "param_grid = [{'clf__C': param_range, \n",
    "               'clf__kernel': ['linear']},\n",
    "                 {'clf__C': param_range, \n",
    "                  'clf__gamma': param_range, \n",
    "                  'clf__kernel': ['rbf']}]\n",
    "\n",
    "gs = GridSearchCV(estimator=pipe_svc, param_grid=param_grid, scoring='accuracy', cv=10,n_jobs=-1)\n",
    "gs = gs.fit(X_train, y_train)\n",
    "print(gs.best_score_)\n",
    "print(gs.best_params_)\n",
    "\n",
    "clf = gs.best_estimator_\n",
    "clf.fit(X_train, y_train)\n",
    "print('Test accuracy: %.3f' % clf.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV accuracy: 0.965 +/- 0.025\n",
      "CV accuracy: 0.921 +/- 0.029\n"
     ]
    }
   ],
   "source": [
    "#Algorithm selection with nested cross-validation\n",
    "\"\"\"\n",
    "Two cross-validation loops are performed in parallel: one by the GridSearchCV estimator to set gamma and the other one by \n",
    "cross_val_score to measure the prediction performance of the estimator. The resulting scores are unbiased estimates of the\n",
    "prediction score on new data.\n",
    "\"\"\"\n",
    "gs = GridSearchCV(estimator=pipe_svc,param_grid=param_grid,scoring='accuracy',cv=2)\n",
    "\n",
    "# Note: Optionally, you could use cv=2 # in the GridSearchCV above to produce # the 5 x 2 nested CV that is shown in the figure.\n",
    "\n",
    "scores = cross_val_score(gs, X_train, y_train, scoring='accuracy', cv=5)\n",
    "print('CV accuracy: %.3f +/- %.3f' % (np.mean(scores), np.std(scores)))\n",
    "\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "gs = GridSearchCV(estimator=DecisionTreeClassifier(random_state=0),\n",
    "                  param_grid=[{'max_depth': [1, 2, 3, 4, 5, 6, 7, None]}],scoring='accuracy',cv=2)\n",
    "\n",
    "scores = cross_val_score(gs, X_train, y_train, scoring='accuracy', cv=5)# to measure the prediction performance of the estimator.\n",
    "print('CV accuracy: %.3f +/- %.3f' % (np.mean(scores), np.std(scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[71  1]\n",
      " [ 2 40]]\n"
     ]
    }
   ],
   "source": [
    "#Reading a confusion matrix\n",
    "\n",
    "\n",
    "from sklearn.metrics import confusion_matrix\n",
    "pipe_svc.fit(X_train, y_train)\n",
    "y_pred = pipe_svc.predict(X_test)\n",
    "confmat = confusion_matrix(y_true=y_test, y_pred=y_pred)\n",
    "print(confmat)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(2.5, 2.5))\n",
    "ax.matshow(confmat, cmap=plt.cm.Blues, alpha=0.3)\n",
    "for i in range(confmat.shape[0]):\n",
    "    for j in range(confmat.shape[1]):\n",
    "        ax.text(x=j, y=i, s=confmat[i, j], va='center', ha='center')\n",
    "\n",
    "plt.xlabel('predicted label')\n",
    "plt.ylabel('true label')\n",
    "\n",
    "plt.tight_layout()\n",
    "# plt.savefig('./figures/confusion_matrix.png', dpi=300)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0], dtype=int64)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "le.transform(['M', 'B'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[71  1]\n",
      " [ 2 40]]\n"
     ]
    }
   ],
   "source": [
    "confmat = confusion_matrix(y_true=y_test, y_pred=y_pred)\n",
    "print(confmat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[40  2]\n",
      " [ 1 71]]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Note that the (true) class 0 samples that are correctly predicted as class 0 (true negatives) are now in the upper left \n",
    "corner of the matrix (index 0, 0). In order to change the ordering so that the true negatives are in the lower right corner\n",
    "(index 1,1) and the true positves are in the upper left, we can use the labels argument like shown below:\n",
    "\"\"\"\n",
    "confmat = confusion_matrix(y_true=y_test, y_pred=y_pred, labels=[1, 0])\n",
    "print(confmat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.976\n",
      "Recall: 0.952\n",
      "F1: 0.964\n"
     ]
    }
   ],
   "source": [
    "#Optimizing the precision and recall of a classification model\n",
    "from sklearn.metrics import precision_score, recall_score, f1_score\n",
    "\n",
    "print('Precision: %.3f' % precision_score(y_true=y_test, y_pred=y_pred))\n",
    "print('Recall: %.3f' % recall_score(y_true=y_test, y_pred=y_pred))\n",
    "print('F1: %.3f' % f1_score(y_true=y_test, y_pred=y_pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.982798668208\n",
      "{'clf__C': 0.1, 'clf__kernel': 'linear'}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import make_scorer\n",
    "\n",
    "scorer = make_scorer(f1_score, pos_label=0)\n",
    "\n",
    "c_gamma_range = [0.01, 0.1, 1.0, 10.0]\n",
    "\n",
    "param_grid = [{'clf__C': c_gamma_range,\n",
    "               'clf__kernel': ['linear']},\n",
    "              {'clf__C': c_gamma_range,\n",
    "               'clf__gamma': c_gamma_range,\n",
    "               'clf__kernel': ['rbf']}]\n",
    "\n",
    "gs = GridSearchCV(estimator=pipe_svc,\n",
    "                  param_grid=param_grid,\n",
    "                  scoring=scorer,\n",
    "                  cv=10,\n",
    "                  n_jobs=-1)\n",
    "gs = gs.fit(X_train, y_train)\n",
    "print(gs.best_score_)\n",
    "print(gs.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  1.79900000e+01   2.06600000e+01   1.17800000e+02 ...,   1.97400000e-01\n",
      "    3.06000000e-01   8.50300000e-02]\n",
      " [  2.02900000e+01   1.43400000e+01   1.35100000e+02 ...,   1.62500000e-01\n",
      "    2.36400000e-01   7.67800000e-02]\n",
      " [  9.00000000e+00   1.44000000e+01   5.63600000e+01 ...,   1.38900000e-02\n",
      "    2.99100000e-01   7.80400000e-02]\n",
      " ..., \n",
      " [  1.72000000e+01   2.45200000e+01   1.14200000e+02 ...,   1.89900000e-01\n",
      "    3.31300000e-01   1.33900000e-01]\n",
      " [  1.40300000e+01   2.12500000e+01   8.97900000e+01 ...,   7.96300000e-02\n",
      "    2.22600000e-01   7.61700000e-02]\n",
      " [  1.30300000e+01   1.84200000e+01   8.26100000e+01 ...,   5.01300000e-02\n",
      "    1.98700000e-01   6.16900000e-02]]\n"
     ]
    }
   ],
   "source": [
    "#Plotting a receiver operating characteristic\n",
    "\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from scipy import interp\n",
    "\n",
    "pipe_lr = Pipeline([('scl', StandardScaler()),\n",
    "                    ('pca', PCA(n_components=2)),\n",
    "                    ('clf', LogisticRegression(penalty='l2', random_state=0,C=100.0))])\n",
    "\n",
    "print X_train\n",
    "X_train2 = X_train[:, [4, 14]]\n",
    "\n",
    "cv = StratifiedKFold(y_train, n_folds=3, random_state=1)\n",
    "\n",
    "fig = plt.figure(figsize=(7, 5))\n",
    "\n",
    "mean_tpr = 0.0\n",
    "mean_fpr = np.linspace(0, 1, 100)\n",
    "all_tpr = []\n",
    "\n",
    "for i, (train, test) in enumerate(cv):\n",
    "    probas = pipe_lr.fit(X_train2[train],y_train[train]).predict_proba(X_train2[test])\n",
    "    fpr, tpr, thresholds = roc_curve(y_train[test],probas[:, 1],pos_label=1)\n",
    "    mean_tpr += interp(mean_fpr, fpr, tpr)\n",
    "    mean_tpr[0] = 0.0\n",
    "    roc_auc = auc(fpr, tpr)\n",
    "    plt.plot(fpr,tpr,lw=1,label='ROC fold %d (area = %0.2f)'% (i+1, roc_auc))\n",
    "\n",
    "plt.plot([0, 1],[0, 1],linestyle='--',color=(0.6, 0.6, 0.6),label='random guessing')\n",
    "\n",
    "mean_tpr /= len(cv)\n",
    "mean_tpr[-1] = 1.0\n",
    "mean_auc = auc(mean_fpr, mean_tpr)\n",
    "plt.plot(mean_fpr, mean_tpr, 'k--',label='mean ROC (area = %0.2f)' % mean_auc, lw=2)\n",
    "plt.plot([0, 0, 1],[0, 1, 1],lw=2,linestyle=':',color='black',label='perfect performance')\n",
    "\n",
    "plt.xlim([-0.05, 1.05])\n",
    "plt.ylim([-0.05, 1.05])\n",
    "plt.xlabel('false positive rate')\n",
    "plt.ylabel('true positive rate')\n",
    "plt.title('Receiver Operator Characteristic')\n",
    "plt.legend(loc=\"lower right\")\n",
    "\n",
    "plt.tight_layout()\n",
    "# plt.savefig('./figures/roc.png', dpi=300)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}