07-Develop-Machine-Learning-Models-RFC.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training (Fitting) Machine Learning Models\n",
    "- Train and Test Models using a Variety of Appropriate [Classification] Algorithms\n",
    "- Tune (Optimize) Each Model''s Hyperparameters\n",
    "- Evaluate the Optimally-Fitted Models Against a Hold-Out (Out-of-Sample) Dataset\n",
    "- Perform K-Fold Cross-Validation Using Each Model\n",
    "- Determine Each Model''s Accuracy and Identify the Best Model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import Required Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SciKit-Learn Version: 0.19.0\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import sklearn\n",
    "print(\"SciKit-Learn Version:\", sklearn.__version__)\n",
    "\n",
    "if sklearn.__version__ >= '0.18.0':  # Starting with sklearn version 18.0\n",
    "    from sklearn.model_selection import train_test_split, cross_val_score\n",
    "    from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "else:\n",
    "    from sklearn.cross_validation import train_test_split, cross_val_score\n",
    "    from sklearn.cross_validation import GridSearchCV, RandomizedSearchCV\n",
    "    \n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, AdaBoostClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report \n",
    "from sklearn.metrics import roc_curve, auc, roc_auc_score\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define Globals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hold_out_percent = 0.2\n",
    "CLASSIFICATION_TYPE = 'Binary'\n",
    "\n",
    "###################################################################################################################\n",
    "# Calculate the Null Accuracy Score: Accommodates Binary or Multiple Classification Type\n",
    "###################################################################################################################\n",
    "def null_accuracy_score(labels, classification_type):  #NOTE: 'labels' must be a pd.Series\n",
    "    if classification_type == CLASSIFICATION_TYPE:\n",
    "        return max(labels[:,].astype(int).mean(), 1 - labels[:,].astype(int).mean())\n",
    "    \n",
    "    elif classification_type == CLASSIFICATION_TYPE:\n",
    "        return labels.value_counts().head(1).item() / len(labels) \n",
    "    \n",
    "    else:\n",
    "        raise ValueError(\"Bad input {0}. Must specify either 'Binary' or 'Multiple'.\".format(classification_type))\n",
    "        \n",
    "\n",
    "##################################################################################################################\n",
    "# Print Evaluation Metrics: Accuracy and Null Accuracy Scores\n",
    "###################################################################################################################\n",
    "def show_accuracy(y_actuals, y_predictions, classification_type):\n",
    "    accuracy = accuracy_score(y_actuals, y_predictions)\n",
    "    null_accuracy = null_accuracy_score(y_actuals, classification_type)\n",
    "\n",
    "    print('----------------------------------------------------------------------------------')\n",
    "    print('Accuracy (The Percentage of Correct Predictions): %0.3f' % accuracy)\n",
    "    print('----------------------------------------------------------------------------------')\n",
    "    print('Null Accuracy (Achieved by Always Predicting the Most Frequent Class): %0.3f' % null_accuracy)\n",
    "    print('----------------------------------------------------------------------------------')   \n",
    "    print('True:', list(y_actuals[0:10]))\n",
    "    print('Pred:', y_predictions[0:10].tolist())\n",
    "    print('----------------------------------------------------------------------------------\\n')\n",
    "        \n",
    "\n",
    "##################################################################################################################\n",
    "# Print Skree Plots: Compare the Explained Variances per Component Between Train/Test and Hold-Out Datasets\n",
    "###################################################################################################################\n",
    "def show_skree_plots(components, variances_explained):\n",
    "    fig, (ax0, ax1) = plt.subplots(nrows=2, sharex=True, figsize=(7, 7))\n",
    "    \n",
    "    ax0.set_title(\"PCA Explained Variances: Train/Test Data\")\n",
    "    ax0.scatter(components, variances_explained[0])\n",
    "    ax0.set_ylabel('Explained Variance (Eigenvalues)')\n",
    "    ax0.grid(True)\n",
    "\n",
    "    ax1.set_title(\"PCA Explained Variances: Hold-Out Data\")\n",
    "    ax1.scatter(components, variances_explained[1])\n",
    "    ax1.set_ylabel('Explained Variance (Eigenvalues)')\n",
    "    ax1.set_xlabel('Number of Components (Eigenvectors)')\n",
    "    ax1.grid(True)\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "        \n",
    "\n",
    "###################################################################################################################\n",
    "# Print a Confusion Matrix:  Accommodates Binary or Multiple Classification Type\n",
    "###################################################################################################################\n",
    "def show_confusion_matrix(y_actuals, y_predictions, y_probabilities, classification_type):\n",
    "    print('-------------------------------------------------------')\n",
    "    print('Confusion Matrix')\n",
    "    print('-------------------------------------------------------')\n",
    "\n",
    "    cm = pd.crosstab(y_actuals, y_predictions, rownames=['Actuals'], colnames=['Predictions'])\n",
    "    print(cm)\n",
    "        \n",
    "    if classification_type == CLASSIFICATION_TYPE:\n",
    "        print('-------------------------------------------------------')\n",
    "        print('Area Under the Curve (AUC): %0.3f' % roc_auc_score(y_actuals, y_probabilities[:, 1]))\n",
    "        \n",
    "    print('-------------------------------------------------------\\n')\n",
    "    \n",
    "    \n",
    "###################################################################################################################\n",
    "# Print a Classification Report\n",
    "###################################################################################################################\n",
    "def show_classification_report(y_actuals, y_predictions):\n",
    "    print('-------------------------------------------------------')\n",
    "    print('Classification Report')\n",
    "    print('-------------------------------------------------------')\n",
    "    print(classification_report(y_actuals, y_predictions))\n",
    "    print('-------------------------------------------------------\\n')\n",
    "    \n",
    "    \n",
    "###################################################################################################################\n",
    "# Print a Classification Report Resulting from K-Fold Cross-Validation\n",
    "###################################################################################################################\n",
    "def show_cv_classification_report(classifier, X, y, K):\n",
    "    accuracy = cross_val_score(classifier, X, y, scoring='accuracy', cv=K)\n",
    "    print('--------------------------------------------------------------------------------------------')\n",
    "    print(\"Accuracy per Fold: \", accuracy)\n",
    "    print(\"Average Accuracy: %0.3f\" % accuracy.mean())\n",
    "    print(\"Standard Deviation of Accuracy: %0.3f\" % accuracy.std())\n",
    "    print('--------------------------------------------------------------------------------------------')\n",
    "    \n",
    "    precision = cross_val_score(classifier, X, y, scoring='precision_weighted', cv=K)\n",
    "    print(\"Precision per Fold: \", precision)\n",
    "    print(\"Average Precision: %0.3f\" % precision.mean())\n",
    "    print(\"Standard Deviation of Precision: %0.3f\" % precision.std())\n",
    "    print('--------------------------------------------------------------------------------------------')\n",
    "    \n",
    "    recall = cross_val_score(classifier, X, y, scoring='recall_weighted', cv=K)\n",
    "    print(\"Recall per Fold: \", recall)\n",
    "    print(\"Average Recall: %0.3f\" % recall.mean())\n",
    "    print(\"Standard Deviation of Recall: %0.3f\" % recall.std())\n",
    "    print('--------------------------------------------------------------------------------------------')\n",
    "    \n",
    "    f1 = cross_val_score(classifier, X, y, scoring='f1_weighted', cv=K)\n",
    "    print(\"F1 per Fold: \", f1)\n",
    "    print(\"Average F1: %0.3f\" % f1.mean())\n",
    "    print(\"Standard Deviation of F1: %0.3f\" % f1.std())\n",
    "    print('--------------------------------------------------------------------------------------------')\n",
    "    \n",
    "\n",
    "###################################################################################################################\n",
    "# Print a Receiver Operating Characteristic (ROC) Curve Plot (Binary Classification Only).\n",
    "###################################################################################################################\n",
    "def show_roc_plot(y_actuals, y_probabilities, classifier_algorithm):\n",
    "    plt.figure(figsize=(5,5))\n",
    "    plt.title('ROC Curve (' + classifier_algorithm + ')')\n",
    "\n",
    "    fpr, tpr, _ = roc_curve(y_actuals, y_probabilities[:, 1])\n",
    "    auc = roc_auc_score(y_actuals, y_probabilities[:, 1])\n",
    "\n",
    "    plt.plot(fpr, tpr, color='darkorange', label='ROC Curve (area = %0.2f)' % auc)\n",
    "    plt.plot([0,1], [0,1], color='steelblue', lw=2, linestyle='--')\n",
    "    plt.xlim([0.0, 1.0])\n",
    "    plt.ylim([0.0, 1.05])\n",
    "    plt.xlabel('False Positive Rate')\n",
    "    plt.ylabel('True Positive Rate')\n",
    "    plt.legend(loc='lower right')\n",
    "    plt.show()\n",
    "    \n",
    "\n",
    "###################################################################################################################\n",
    "# Save a Trained Model to a Pickle File.\n",
    "###################################################################################################################\n",
    "def export_model(model, algorithm_name):\n",
    "    file_name = \"Customer_Churn_{0}.pkl\".format(algorithm_name)\n",
    "    file_target = open(os.path.join(os.getcwd(), file_name), 'wb')\n",
    "    pickle.dump(rfc, file_target, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    file_target.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DataSet Dimensions: (7043, 20)\n"
     ]
    }
   ],
   "source": [
    "# Locate the Data File to be Ingested.\n",
    "data_dir = os.path.join(os.getcwd(), 'Data')\n",
    "source_file = os.path.join(data_dir, 'WA-Telco-Customer-Churn-ML.xlsx')\n",
    "\n",
    "# Read the Data from the Source File\n",
    "df = pd.read_excel(source_file, header=0)\n",
    "\n",
    "# Drop the Unique Identifier\n",
    "df = df.drop(labels=['customerID'], axis=1)\n",
    "\n",
    "# Display the Dimensions of the DataSet Being Used.\n",
    "print(\"DataSet Dimensions:\", df.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Make Appropriate Data Type Assignments."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "gender              category\n",
       "SeniorCitizen       category\n",
       "Partner             category\n",
       "Dependents          category\n",
       "tenure                  int8\n",
       "PhoneService        category\n",
       "MultipleLines       category\n",
       "InternetService     category\n",
       "OnlineSecurity      category\n",
       "OnlineBackup        category\n",
       "DeviceProtection    category\n",
       "TechSupport         category\n",
       "StreamingTV         category\n",
       "StreamingMovies     category\n",
       "Contract            category\n",
       "PaperlessBilling    category\n",
       "PaymentMethod       category\n",
       "MonthlyCharges       float32\n",
       "TotalCharges         float32\n",
       "Churn               category\n",
       "dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.gender = df.gender.astype('category')\n",
    "df.SeniorCitizen = df.SeniorCitizen.astype('category')\n",
    "df.Partner = df.Partner.astype('category')\n",
    "df.Dependents = df.Dependents.astype('category')\n",
    "df.tenure = pd.to_numeric(df.tenure, downcast='integer', errors='coerce')\n",
    "df.PhoneService = df.PhoneService.astype('category')\n",
    "df.MultipleLines = df.MultipleLines.astype('category')\n",
    "df.InternetService = df.InternetService.astype('category')\n",
    "df.OnlineSecurity = df.OnlineSecurity.astype('category')\n",
    "df.OnlineBackup = df.OnlineBackup.astype('category')\n",
    "df.DeviceProtection = df.DeviceProtection.astype('category')\n",
    "df.TechSupport = df.TechSupport.astype('category')\n",
    "df.StreamingTV = df.StreamingTV.astype('category')\n",
    "df.StreamingMovies = df.StreamingMovies.astype('category')\n",
    "df.Contract = df.Contract.astype('category')\n",
    "df.PaperlessBilling = df.PaperlessBilling.astype('category')\n",
    "df.PaymentMethod = df.PaymentMethod.astype('category')\n",
    "df.MonthlyCharges = pd.to_numeric(df.MonthlyCharges, downcast='float', errors='coerce')\n",
    "df.TotalCharges = pd.to_numeric(df.TotalCharges, downcast='float', errors='coerce')\n",
    "df.Churn = df.Churn.astype('category')\n",
    "df.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Separate the Independant (Predictor) Variables (X) from the Dependant (Target, Response, or Label) Variable (y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = df.drop(labels=['Churn'], axis=1)\n",
    "y = df.Churn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Select Features Based on Feature Selection Exercises"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = df[['TotalCharges','MonthlyCharges','tenure','Contract',\n",
    "        'PaymentMethod'\n",
    "        #,'OnlineSecurity'\n",
    "        , 'InternetService'\n",
    "    ]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train a Baseline Classifier Using K-Nearest Neighbors and K-Fold Cross-Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "knn = KNeighborsClassifier()\n",
    "show_cv_classification_report(knn, X, y, 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Perform Hyper-Parameter Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Define the parameter values to be searched\n",
    "k_range = list(range(2, 26))\n",
    "weight_options = ['uniform','distance']\n",
    "\n",
    "# Create a parameter grid\n",
    "param_grid = dict(n_neighbors=k_range, weights=weight_options)\n",
    "\n",
    "# Instantiate the grid\n",
    "grid = GridSearchCV(knn, param_grid, cv=10, scoring='accuracy', return_train_score=False, n_jobs=-1)\n",
    "\n",
    "# Finally, fit the grid\n",
    "grid.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### View the Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(grid.cv_results_)[['mean_test_score', 'std_test_score', 'params']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mean_test_scores = grid.cv_results_['mean_test_score']\n",
    "params = grid.cv_results_['params']\n",
    "\n",
    "items = list(range(0, len(params)))\n",
    "weights = []\n",
    "\n",
    "for i in items:\n",
    "    weights.append(params[i]['weights'])\n",
    "\n",
    "results = pd.DataFrame({'scores':mean_test_scores, 'weights':weights})      \n",
    "uniform_scores = results[results.weights == 'uniform'].scores\n",
    "distance_scores = results[results.weights == 'distance'].scores\n",
    "\n",
    "print('----------------------------------------------------------------------------------')\n",
    "print('Mean Test Scores (Uniform):', round(uniform_scores, 3).tolist())\n",
    "print('----------------------------------------------------------------------------------')\n",
    "print('Mean Test Scores (Distance):', round(distance_scores, 3).tolist())\n",
    "print('----------------------------------------------------------------------------------')\n",
    "print('K-Range Values:', k_range)\n",
    "print('----------------------------------------------------------------------------------')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plot the Mean Test Scores by Weight Against the Number of Neighbors (K)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Plot the scores\n",
    "plt.title('Mean Test Scores by Weight')\n",
    "plt.plot(k_range, uniform_scores, color='steelblue', label='Uniform')\n",
    "plt.plot(k_range, distance_scores, color='darkorange', label='Distance')\n",
    "plt.legend(loc='lower right', title='Weights')\n",
    "plt.xlabel('Value of K for KNN')\n",
    "plt.ylabel('Cross-validated Accuracy')\n",
    "plt.grid(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Examine the best score\n",
    "print(grid.best_score_)\n",
    "print(grid.best_params_)\n",
    "print(grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Use the Best Hyper-Parameters to Make Predictions Using Hold-Out Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "n_neighbors = grid.best_params_['n_neighbors']\n",
    "weights_option = grid.best_params_['weights']\n",
    "\n",
    "knn = KNeighborsClassifier(n_neighbors = n_neighbors, weights = weights_option)\n",
    "show_cv_classification_report(knn, X, y, 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Partition Dataset for Training, Testing and Evaluating Models\n",
    "#### Create a Hold-Out Data Set for Model Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Divide into Train/Test and Evaluation (Hold-Out) Sets.\n",
    "X, X_eval, y, y_eval = train_test_split(X, y, test_size = hold_out_percent, random_state=42)\n",
    "\n",
    "# Use 80% of the observations for training and testing...\n",
    "print('-------------------------------------------------------------------')\n",
    "print(\"Train/Test Observations:\", X.shape)\n",
    "\n",
    "# ...And hod out the rest for final evaluation.\n",
    "print(\"Evaluation Observations:\", X_eval.shape)\n",
    "print('-------------------------------------------------------------------\\n')\n",
    "\n",
    "# Examine the distribution of Labels for the two datasets.\n",
    "print(\"Train/Test Label Distribution:\\n\", y.value_counts())\n",
    "print('-------------------------------------------------------------------')\n",
    "print(\"Evaluation Label Distribution:\\n\", y_eval.value_counts())\n",
    "print('-------------------------------------------------------------------')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Training and Testing Datasets "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# Use 60% of the observations for training...\n",
    "print('-------------------------------------------------------------------')\n",
    "print(\"Training Observations:\", X_train.shape)\n",
    "\n",
    "# ...And use the rest for testing.\n",
    "print(\"Testing Observations:\", X_test.shape)\n",
    "print('-------------------------------------------------------------------\\n')\n",
    "\n",
    "# Examine the distribution of Labels for the two datasets.\n",
    "print(\"Training Label Distribution:\\n\", y_train.value_counts())\n",
    "print('-------------------------------------------------------------------')\n",
    "print(\"Testing Label Distribution:\\n\", y_test.value_counts())\n",
    "print('-------------------------------------------------------------------')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Perform Scalar Normalization of Each Dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sc = StandardScaler()\n",
    "\n",
    "X_train = sc.fit_transform(X_train)\n",
    "X_test = sc.transform(X_test)\n",
    "\n",
    "X = sc.fit_transform(X)\n",
    "X_eval = sc.transform(X_eval)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train a Classification Model Using the RandomForestsClassifier "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Train using the Training data.\n",
    "rfc = RandomForestClassifier(random_state=42)\n",
    "rfc.fit(X_train, y_train)\n",
    "\n",
    "# Make Predictions using the Test data.\n",
    "y_pred = rfc.predict(X_test)\n",
    "y_probs = rfc.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluate the Model's Efficacy using Metrics, a Confusion Matrix and Classification Report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "show_accuracy(y_test, y_pred, CLASSIFICATION_TYPE)\n",
    "show_confusion_matrix(y_test, y_pred, y_probs, CLASSIFICATION_TYPE)\n",
    "show_classification_report(y_test, y_pred)\n",
    "\n",
    "if CLASSIFICATION_TYPE == 'Binary':\n",
    "    show_roc_plot(y_test, y_probs, 'Random Forests Classifier')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Perform Hyper-Parameter Tuning for the Random Forests Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Define the parameter values to be searched\n",
    "estimator_range = list(range(2, 21))\n",
    "criterion_options = ['gini','entropy']\n",
    "max_features_options = [None, 'auto','sqrt','log2']\n",
    "max_depth_range = list(range(2, 21))\n",
    "\n",
    "# Create parameter distributions\n",
    "param_dist = dict(n_estimators = estimator_range\n",
    "                  , max_features = max_features_options\n",
    "                  , criterion = criterion_options\n",
    "                  , max_depth = max_depth_range\n",
    "                 )\n",
    "\n",
    "# Instantiate the grid\n",
    "rand = RandomizedSearchCV(rfc, param_dist, cv=10, scoring='accuracy', n_iter=10, random_state=42, n_jobs=-1) \n",
    "rand.return_train_score=False\n",
    "\n",
    "# Finally, fit the grid\n",
    "rand.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### View the Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.DataFrame(rand.cv_results_)[['mean_test_score', 'std_test_score', 'params']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dist_mean_scores = rand.cv_results_['mean_test_score']\n",
    "param_estimators = rand.cv_results_['params']\n",
    "items = list(range(0, len(param_estimators)))\n",
    "\n",
    "estimators = []\n",
    "\n",
    "for i in items:\n",
    "    estimators.append(param_estimators[i]['n_estimators'])\n",
    "    estimators.sort()\n",
    "    \n",
    "print('-------------------------------------------------------------------')    \n",
    "print('Mean Test Scores:', dist_mean_scores)\n",
    "print('-------------------------------------------------------------------')\n",
    "print('Estimator Values:', estimators)\n",
    "print('-------------------------------------------------------------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Plot the scores\n",
    "plt.title('Mean Test Scores by Number of Estimators')\n",
    "plt.plot(estimators, dist_mean_scores, color='steelblue')\n",
    "plt.xlabel('Number of Estimators')\n",
    "plt.ylabel('Mean Test Scores')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Examine the best score\n",
    "print(rand.best_score_)\n",
    "print(rand.best_params_)\n",
    "print(rand.best_estimator_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Use the Best Parameters to Make Predictions Using the Hold-Out Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "n_estimators = rand.best_params_['n_estimators']\n",
    "max_feature = rand.best_params_['max_features']\n",
    "max_depth = rand.best_params_['max_depth']\n",
    "criteria = rand.best_params_['criterion']\n",
    "\n",
    "rfc = RandomForestClassifier(n_estimators = n_estimators\n",
    "                             , max_features = max_feature\n",
    "                             , criterion = criteria\n",
    "                             , max_depth = max_depth\n",
    "                             , random_state = 42\n",
    "                            )\n",
    "rfc.fit(X, y)\n",
    "\n",
    "y_pred = rand.predict(X_eval)\n",
    "y_probs = rand.predict_proba(X_eval)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluate the Model's Efficacy using Metrics, a Confusion Matrix and Classification Report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "show_accuracy(y_eval, y_pred, CLASSIFICATION_TYPE)\n",
    "show_confusion_matrix(y_eval, y_pred, y_probs, CLASSIFICATION_TYPE)\n",
    "show_classification_report(y_eval, y_pred)\n",
    "\n",
    "if CLASSIFICATION_TYPE == 'Binary':\n",
    "    show_roc_plot(y_eval, y_probs, 'Random Forests Classifier')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Use K-Fold Cross-Validation to Detect Overfitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "show_cv_classification_report(rfc, X, y, 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Export the Best Fit Model to a Pickle file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "export_model(rfc, \"RandomForestsClassifier\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}