marcio_workshop.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "nbpresent": {
     "id": "f77e2465-07f3-4ad0-ab0b-970cc19da161"
    }
   },
   "source": [
    "# <center> AN INTRODUCTION TO MACHINE LEARNING\n",
    "## <center> CSCAR WORKSHOP <br/><br/> 03/22/2018\n",
    "## <center> Marcio Mourao and Michael Clark"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# <center> Setup for Anaconda / Jupyter Notebook\n",
    "\n",
    "<ul>\n",
    "    <li>Go to the page https://marcio-mourao.github.io/</li>\n",
    "    <li>Download the materials (first two docs) under \"\" to your \"username/Documents\"</li><br/>    \n",
    "    <li>Click the Windows button (Bottom Left Corner)</li>\n",
    "    <li>Click \"All apps\"</li>\n",
    "    <li>Click \"Anaconda3\"</li>\n",
    "    <li>Click \"Jupyter Notebook\" </li>\n",
    "    <li>Click \"Workshop.ipynb\" (this should open a new tab in the browser)</li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <center> References\n",
    "\n",
    "<ul>\n",
    "  <li>https://www.continuum.io/anaconda-overview</li>\n",
    "  <li>http://www.numpy.org/</li>\n",
    "  <li>http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html</li>\n",
    "  <li>http://pandas.pydata.org/pandas-docs/stable/10min.html</li>\n",
    "  <li>http://matplotlib.org/</li>\n",
    "  <li>http://scikit-learn.org/stable/documentation.html</li>\n",
    "  <li>https://pypi.python.org/pypi/patsy</li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Note 1 : This document does not do any pre-processing of the data (data is completely clean)\n",
    "##### Note 2 : This document assumes binary classification tasks "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Import relevant general modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "nbpresent": {
     "id": "56b6dfff-16a3-4ab8-b0cb-497226d0e34f"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "print(sys.version)\n",
    "print(np.__version__)\n",
    "print(pd.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Choice of dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "choice_dataset = 1 # Index of choice for the dataset names below - choose one dataset\n",
    "dataset_names = ['wine', 'adult'] # List of available datasets\n",
    "dv = ['good', 'income']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Load and describe the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Creates a dataframe from the data in the file\n",
    "data_df = pd.read_csv(dataset_names[choice_dataset] + '.csv', na_values=['?'])\n",
    "data_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Obtains the number of lines and columns of the dataframe\n",
    "data_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Obtains the dataframe main types\n",
    "data_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Provides a statistical summary of the data\n",
    "data_df.describe(include='all')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup modules, functions and data for Machine Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Import scikit-learn and patsy modules\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "from sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn import metrics\n",
    "from patsy import dmatrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Import plotly modules\n",
    "from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\n",
    "import plotly.graph_objs as go\n",
    "\n",
    "init_notebook_mode(connected=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#This function returns model predictions\n",
    "def get_model_predictions(model):   \n",
    "    y_pred_prob = model.predict_proba(X_test) #Obtain probability predictions\n",
    "    y_pred_class = model.predict(X_test) #Obtain class predictions\n",
    "    output_df = pd.DataFrame(y_pred_prob, columns=['Prob Class A', 'Prob Class B'])\n",
    "    output_df['Predicted Class'] = y_pred_class\n",
    "    \n",
    "    return output_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#This function plots values of a dataframe\n",
    "def plot_table(df, width, height):\n",
    "    trace = go.Table(\n",
    "        header=dict(values = df.columns,\n",
    "                    line = dict(color='#7D7F80'),\n",
    "                    fill = dict(color='#a1c3d1'),\n",
    "                    align = ['center'] * 5),\n",
    "        cells=dict(values = [df[col] for col in df.columns],\n",
    "                   line = dict(color='#7D7F80'),\n",
    "                   fill = dict(color='#EDFAFF'),\n",
    "                   align = ['center'] * 5))\n",
    "    \n",
    "    layout = dict(width=width, height=height)\n",
    "    fig = dict(data=[trace], layout = layout)\n",
    "    iplot(fig, filename = 'styled_table')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#This function obtains the formula to construct design matrices\n",
    "def get_formula(dataset_name):\n",
    "    if dataset_name=='adult':\n",
    "        #Set formula to use in dmatrices\n",
    "        formula = 'income ~ -1 + age + workclass + educationnum + maritalstatus + ' + \\\n",
    "                   'occupation + relationship + race + sex + ' + \\\n",
    "                   'capitalgain + capitalloss + hoursperweek + nativecountry'\n",
    "    elif dataset_name=='wine':\n",
    "        #Set formula to use in dmatrices\n",
    "        formula = 'good ~ -1 + fixedacidity + volatileacidity + citricacid + residualsugar + ' + \\\n",
    "                   'chlorides + totalsulfurdioxide + pH + sulphates + alcohol'\n",
    "    \n",
    "    return formula"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#This function plots a roc curve\n",
    "def plot_roc_curve(fpr, tpr, roc_auc):\n",
    "    plt.figure()\n",
    "    colors = ['r', 'g', 'b', 'm', 'c']\n",
    "    for i, color in zip(range(len(fpr)), colors):\n",
    "        plt.plot(fpr[i], tpr[i], color=colors[i], lw=2, \n",
    "                 label='ROC curve of model {0} (area = {1:0.2f})'\n",
    "                 ''.format(i, roc_auc[i]))\n",
    "\n",
    "    plt.plot([0, 1], [0, 1], color='k', lw=2, linestyle='--')\n",
    "    plt.xlim([0.0, 1.0])\n",
    "    plt.ylim([0.0, 1.05])\n",
    "    plt.xlabel('False Positive Rate')\n",
    "    plt.ylabel('True Positive Rate')\n",
    "    plt.title('Receiver operating characteristic')\n",
    "    plt.legend(loc=\"lower right\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Rename columns of the dataframe for compatibility with patsy\n",
    "data_df.columns = [col.replace('.','') for col in data_df.columns] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Converts column income into an integer\n",
    "data_df[dv[choice_dataset]] = pd.factorize(data_df[dv[choice_dataset]])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Select numerical columns for later scaling\n",
    "numericalColumns = data_df.select_dtypes(exclude=['object']).columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Obtain the design matrix\n",
    "y, X = dmatrices(get_formula(dataset_names[choice_dataset]), data_df, return_type = 'dataframe')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#The dependent variable needs to be a unidimensional vector rather than a dataframe\n",
    "y = y[dv[choice_dataset]].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Obtain the data for the fitting\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1234, test_size=.2, stratify = y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Scale both X_train and X_test datasets separately\n",
    "scaleColumns = numericalColumns & X.columns\n",
    "X_train = X_train.copy()\n",
    "X_test = X_test.copy()\n",
    "X_train[scaleColumns] = StandardScaler().fit_transform(X_train[scaleColumns])\n",
    "X_test[scaleColumns] = StandardScaler().fit_transform(X_test[scaleColumns])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Choice of models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Select one or more of the following models -> 'KNN', 'LR', RF', 'SVM', 'NN'\n",
    "modelsStr = ['RF']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Machine Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Models\n",
    "KNN = KNeighborsClassifier()\n",
    "LR = LogisticRegression()\n",
    "RF = RandomForestClassifier(n_estimators=10, criterion='gini')\n",
    "SVM = SVC(kernel='rbf', probability=True)\n",
    "NN = MLPClassifier(activation='logistic', learning_rate='constant')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Options for the models\n",
    "KNNOpts = {'n_neighbors': np.square(np.arange(2,6))}\n",
    "LROpts = {'C': np.arange(1,5)}\n",
    "RFOpts = {'max_features': np.arange(2,7)}\n",
    "SVMOpts = {'C': [.25, .5, 1, 2, 4]}\n",
    "NNOpts = {'hidden_layer_sizes': [(3,), (5,), (7,), (9,)], 'alpha': np.linspace(.0001, 1, 10)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Performs search on a grid\n",
    "gridsCV = [GridSearchCV(eval(modelStr), cv=10, param_grid=eval(modelStr+'Opts'), return_train_score=True) for modelStr in modelsStr]\n",
    "modelsGrid = [gridCV.fit(X_train, y_train) for gridCV in gridsCV]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Displays the best cross-validation score of all grid points\n",
    "[print(np.round(modelGrid.best_score_,2)) for modelGrid in modelsGrid]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Obtain the model predictions (both probabilities and classes in one single dataframe)\n",
    "y_preds = [get_model_predictions(modelGrid) for modelGrid in modelsGrid]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Display the predictions for each observation or case\n",
    "[plot_table(np.round(y_pred.head(10),2), 500, 500) for y_pred in y_preds]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Display the accuracy score\n",
    "[print('Model Accuracy Score: ', np.round(metrics.accuracy_score(y_test, y_pred['Predicted Class']),2)) for y_pred in y_preds]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Obtains and displays classification reports\n",
    "crs = [metrics.classification_report(y_test, np.array(y_pred['Predicted Class'])) for y_pred in y_preds]\n",
    "[print(cr) for cr in crs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Obtains and displays the confusion matrix\n",
    "cms = [metrics.confusion_matrix(y_test, np.array(y_pred['Predicted Class'])) for y_pred in y_preds]\n",
    "cms_df = [pd.DataFrame(cm, columns = ['Predicted Class A','Predicted Class B']) for cm in cms]\n",
    "cms_df = [cm_df/cm_df.sum().sum() for cm_df in cms_df]\n",
    "cms_df = [pd.concat([pd.DataFrame(['Real Class A', 'Real Class B']), cm_df], axis=1) for cm_df in cms_df]\n",
    "[plot_table(np.round(cm_df,2), 500, 500) for cm_df in cms_df]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Plot roc curves for the models\n",
    "y_scores = [y_pred['Prob Class B'].values for y_pred in y_preds]\n",
    "\n",
    "fpr = dict()\n",
    "tpr = dict()\n",
    "roc_auc = dict()\n",
    "for i in range(len(y_scores)):\n",
    "    fpr[i], tpr[i], _ = metrics.roc_curve(y_test, y_scores[i], pos_label=None, sample_weight=None, drop_intermediate=True)\n",
    "    roc_auc[i] = metrics.auc(fpr[i], tpr[i])\n",
    "    \n",
    "plot_roc_curve(fpr, tpr, roc_auc)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  },
  "nbpresent": {
   "slides": {
    "8ce21dab-2b7a-439c-8f0a-8a1036a1ac0d": {
     "id": "8ce21dab-2b7a-439c-8f0a-8a1036a1ac0d",
     "prev": null,
     "regions": {
      "d8401624-efe9-4853-a676-de863b4ba3cd": {
       "attrs": {
        "height": 1,
        "width": 1,
        "x": 0,
        "y": 0
       },
       "id": "d8401624-efe9-4853-a676-de863b4ba3cd"
      }
     },
     "theme": null
    }
   },
   "themes": {}
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}