Notebook_3_Labeling_FeatureSelection_Modeling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Notebook #3\n",
    "In this notebook, we will demonstrate the following things:  \n",
    "-  ***Over-labeling technique especially for predictive maintenance use cases***\n",
    "-  ***Feature reduction using PCA***\n",
    "-  ***Prepare train and test data***\n",
    "-  ***Binary classification Model***\n",
    "    1.  *Random Forest*\n",
    "    2.  *Gradient Boosted Tree*\n",
    "    3.  *Hyper-parameter tuning of Random Forest model and Cross-Validation*\n",
    " <br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/lib/python2.7/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "/anaconda/lib/python2.7/site-packages/sklearn/grid_search.py:43: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pyspark.sql.functions as F\n",
    "import time\n",
    "import pandas as pd\n",
    "import subprocess\n",
    "import sys\n",
    "import os\n",
    "import re\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import datetime\n",
    "import atexit\n",
    "\n",
    "from pyspark import SparkConf\n",
    "from pyspark import SparkContext\n",
    "from pyspark import SQLContext\n",
    "from pyspark.sql.types import *\n",
    "from pyspark.sql.functions import col,udf,lag,date_add,explode,lit,concat,unix_timestamp,sum, abs\n",
    "from pandas import DataFrame\n",
    "from pyspark.sql.dataframe import *\n",
    "from pyspark.ml.classification import *\n",
    "from pyspark.ml.feature import VectorAssembler\n",
    "from pyspark.sql.window import Window\n",
    "from pyspark.sql.types import DateType\n",
    "from datetime import datetime, timedelta\n",
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.feature import StringIndexer, OneHotEncoder\n",
    "from pyspark.ml.evaluation import *\n",
    "from pyspark.ml.tuning import CrossValidator\n",
    "from pyspark.ml.tuning import ParamGridBuilder\n",
    "from pyspark.sql import Row\n",
    "from pyspark.ml import Pipeline, PipelineModel\n",
    "from pyspark.mllib.evaluation import BinaryClassificationMetrics\n",
    "from pyspark.ml.feature import OneHotEncoder, StringIndexer, VectorIndexer, RFormula\n",
    "from pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n",
    "from sklearn.metrics import roc_curve,auc\n",
    "from pyspark.sql.functions import month, weekofyear, dayofmonth\n",
    "from pyspark.ml.feature import ChiSqSelector\n",
    "from pyspark.ml.linalg import Vectors\n",
    "from pyspark.ml.feature import StandardScaler\n",
    "from pyspark.ml.feature import PCA\n",
    "from pyspark.ml.feature import MinMaxScaler\n",
    "from pyspark.sql.types import DoubleType\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
    "from pyspark.ml.classification import GBTClassifier\n",
    "from sklearn.metrics import precision_recall_fscore_support as score\n",
    "from sklearn.grid_search import ParameterGrid\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load result from Notebook #2 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2085102, 1277)\n",
      "+--------------------+--------+\n",
      "|                 key|deviceid|\n",
      "+--------------------+--------+\n",
      "|N0001_2012-12-08 ...|   N0001|\n",
      "|N0001_2012-12-09 ...|   N0001|\n",
      "|N0001_2012-12-10 ...|   N0001|\n",
      "+--------------------+--------+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# load result from Notebook #2 (\"FeatureEngineering_RollingCompute\")\n",
    "df = sqlContext.read.parquet('/mnt/resource/PysparkExample/notebook2_result.parquet')\n",
    "\n",
    "# check the dimension of the dataset and make sure things look right\n",
    "print(df.count(), len(df.columns))\n",
    "df.select('key','deviceid').show(3)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Over-Lableing Technique\n",
    "**For predictive maintenance use cases, we usually want to predict failure/problem ahead of time. For example, in our example, we would like to be able to predict a customer complain 7 days in advance. That means for the label column, we need to label all the 7 days before the actual failure/problem day as \"1\".***\n",
    "<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 6, 3, 5, 4, 8, 7, 2, 0]\n"
     ]
    }
   ],
   "source": [
    "#------------------------------------ Create label column ------------------------------------#\n",
    "\n",
    "# Step 1: \n",
    "df = df.withColumn('label_tmp', col('problemreported')) \n",
    "\n",
    "# Step 2:\n",
    "wSpec = Window.partitionBy('deviceid').orderBy(df.date.desc())\n",
    "lag_window = 7  # Define how many days in advance we want to predict failure\n",
    "\n",
    "for i in range(lag_window):\n",
    "    lag_values = lag(df.label_tmp, default=0).over(wSpec)\n",
    "    df = df.withColumn('label_tmp', F.when((col('label_tmp')==1) | (lag_values==None) | (lag_values<1) | (lag_values>=(lag_window+1)), col('label_tmp')).otherwise(lag_values+1))\n",
    "\n",
    "# check the results\n",
    "print(df.select('label_tmp').distinct().rdd.map(lambda r: r[0]).collect()) \n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+---------------+---------+-----+\n",
      "|deviceid|                date|problemreported|label_tmp|label|\n",
      "+--------+--------------------+---------------+---------+-----+\n",
      "|   N0001|2012-12-08 00:00:...|              0|        0|  0.0|\n",
      "|   N0001|2012-12-09 00:00:...|              0|        0|  0.0|\n",
      "|   N0001|2012-12-10 00:00:...|              0|        0|  0.0|\n",
      "|   N0001|2012-12-11 00:00:...|              0|        0|  0.0|\n",
      "|   N0001|2012-12-12 00:00:...|              0|        0|  0.0|\n",
      "|   N0001|2012-12-13 00:00:...|              0|        0|  0.0|\n",
      "|   N0001|2012-12-14 00:00:...|              0|        0|  0.0|\n",
      "|   N0001|2012-12-15 00:00:...|              0|        0|  0.0|\n",
      "|   N0001|2012-12-16 00:00:...|              0|        0|  0.0|\n",
      "|   N0001|2012-12-17 00:00:...|              0|        8|  1.0|\n",
      "|   N0001|2012-12-18 00:00:...|              0|        7|  1.0|\n",
      "|   N0001|2012-12-19 00:00:...|              0|        6|  1.0|\n",
      "|   N0001|2012-12-20 00:00:...|              0|        5|  1.0|\n",
      "|   N0001|2012-12-21 00:00:...|              0|        4|  1.0|\n",
      "|   N0001|2012-12-22 00:00:...|              0|        3|  1.0|\n",
      "|   N0001|2012-12-23 00:00:...|              0|        2|  1.0|\n",
      "|   N0001|2012-12-24 00:00:...|              1|        1|  1.0|\n",
      "|   N0001|2012-12-25 00:00:...|              0|        0|  0.0|\n",
      "|   N0001|2012-12-26 00:00:...|              0|        0|  0.0|\n",
      "|   N0001|2012-12-27 00:00:...|              0|        0|  0.0|\n",
      "+--------+--------------------+---------------+---------+-----+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Step 3:\n",
    "### please note that we need to make \"label\" column double instead of integer for the pyspark classification models \n",
    "df = df.withColumn('label', F.when(col('label_tmp') > 0, 1.0).otherwise(0.0))\n",
    "df.createOrReplaceTempView(\"df_view\") \n",
    " \n",
    "# Step 4:\n",
    "df.orderBy('deviceid', 'date').select('deviceid', 'date', 'problemreported', 'label_tmp', 'label').show(20) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-------------------+\n",
      "|summary|              label|\n",
      "+-------+-------------------+\n",
      "|  count|            2085102|\n",
      "|   mean|0.01456187754843648|\n",
      "| stddev|0.11979080162007248|\n",
      "|    min|                0.0|\n",
      "|    max|                1.0|\n",
      "+-------+-------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Visualize the distribution of \"label\" column\n",
    "df.select('label').describe().show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Reduction\n",
    "-  There are not many packages for feature selection in PySpark 2.0.2.,so we decided to use PCA to reduce the demensionality.\n",
    "-  There are so many features especially rolling features, we need to perform feature selection to reduce the feature set size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1150"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## check the number of rolling features\n",
    "len([col_n for col_n in df.columns if '_rolling' in col_n])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.0720783785227,0.0329825870736,0.027901485354,0.0243662075346,0.0231109612038,0.0230696286064,0.0175345239922,0.0169050432163,0.0160163382589,0.0151685106261,0.0139611893027,0.0132999118492,0.0128137208726,0.012460010309,0.0123226258655,0.0121766225655,0.0110137480593,0.0106161694018,0.0103270115698,0.00977105423534,0.00908164998253,0.00880547728788,0.0083968875896,0.00833527942731,0.00816512952224,0.00807362710216,0.00787447043078,0.00782728526041,0.00776590544977,0.00753956685426,0.00735611311084,0.00717316781783,0.00710561994143,0.00694768452248,0.0068423949855,0.00669806943767,0.00660469671941,0.00658491264526,0.00646747772504,0.00638036477672,0.00630822833773,0.00628238685298,0.00608372864476,0.00606093314036,0.00604486211127,0.00599575013941,0.00591032102729,0.0057055133175,0.00568659059503,0.00551330593664]\n",
      "+--------------------+--------------------+\n",
      "|                 key|   pca_roll_features|\n",
      "+--------------------+--------------------+\n",
      "|N0001_2012-12-08 ...|[0.0,0.0,0.0,0.0,...|\n",
      "|N0001_2012-12-09 ...|[0.0,0.0,0.0,0.0,...|\n",
      "|N0001_2012-12-10 ...|[0.0,0.0,0.0,0.0,...|\n",
      "|N0001_2012-12-11 ...|[0.0,0.0,0.0,0.0,...|\n",
      "|N0001_2012-12-12 ...|[0.0,0.0,0.0,0.0,...|\n",
      "+--------------------+--------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Step 1\n",
    "# Use RFormula to create the feature vector\n",
    "rolling_features = list(s for s in df.columns if \"_rolling\" in s)\n",
    "formula = RFormula(formula = \"~\" + \"+\".join(rolling_features))\n",
    "output = formula.fit(df).transform(df).select(\"key\",\"features\") \n",
    "\n",
    "\n",
    "# Step 2 \n",
    "# Before PCA, we need to standardize the features, it is very important...\n",
    "# We compared 1) standardization, 2) min-max normalization, 3) combintion of standardization and min-max normalization\n",
    "# In 2), the 1st PC explained more than 67% of the variance\n",
    "# 1) & 3) generate exactly the same results for model.explainedVariance. \n",
    "# That means min-max normalization does not help in our case\n",
    "\n",
    "scaler = StandardScaler(inputCol=\"features\", outputCol=\"scaledFeatures\",\n",
    "                        withStd=True, withMean=False)\n",
    "\n",
    "# Compute summary statistics by fitting the StandardScaler\n",
    "scalerModel = scaler.fit(output)\n",
    "\n",
    "# Normalize each feature to have unit standard deviation.\n",
    "scaledData = scalerModel.transform(output)\n",
    "\n",
    "\n",
    "# Step 3\n",
    "pca = PCA(k=50, inputCol=\"scaledFeatures\", outputCol=\"pca_roll_features\")\n",
    "model = pca.fit(scaledData)\n",
    "result = model.transform(scaledData).select(\"key\",\"pca_roll_features\")\n",
    "print(model.explainedVariance)\n",
    "\n",
    "\n",
    "# Step 4\n",
    "df = df.join(result, 'key', 'inner')\n",
    "rolling_drop_list = [col_n for col_n in df.columns if '_rolling' in col_n]\n",
    "df = df.select([column for column in df.columns if column not in rolling_drop_list])\n",
    "\n",
    "df.select('key','pca_roll_features').show(5)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare train and test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define list of input columns for downstream modeling\n",
    "input_features = [\n",
    " 'usage_count_1',\n",
    " 'usage_count_2',\n",
    " 'problem_type_1',\n",
    " 'problem_type_2',\n",
    " 'problem_type_3',\n",
    " 'problem_type_4',\n",
    " 'error_count_1',\n",
    " 'error_count_2',\n",
    " 'error_count_3',\n",
    " 'error_count_4',\n",
    " 'error_count_5',\n",
    " 'error_count_6',\n",
    " 'error_count_7',\n",
    " 'error_count_8',\n",
    " 'month',\n",
    " 'weekofyear',\n",
    " 'dayofmonth',\n",
    " 'warn_type1_total',\n",
    " 'warn_type2_total',\n",
    " 'fault_code_type_1_count',\n",
    " 'fault_code_type_2_count',\n",
    " 'fault_code_type_3_count',\n",
    " 'fault_code_type_4_count',\n",
    " 'problem_type_1_per_usage1',\n",
    " 'problem_type_2_per_usage1',\n",
    " 'problem_type_3_per_usage1',\n",
    " 'problem_type_4_per_usage1',\n",
    " 'fault_code_type_1_count_per_usage1',\n",
    " 'fault_code_type_2_count_per_usage1',\n",
    " 'fault_code_type_3_count_per_usage1',\n",
    " 'fault_code_type_4_count_per_usage1',\n",
    " 'problem_type_1_per_usage2',\n",
    " 'problem_type_2_per_usage2',\n",
    " 'problem_type_3_per_usage2',\n",
    " 'problem_type_4_per_usage2',\n",
    " 'fault_code_type_1_count_per_usage2',\n",
    " 'fault_code_type_2_count_per_usage2',\n",
    " 'fault_code_type_3_count_per_usage2',\n",
    " 'fault_code_type_4_count_per_usage2',   \n",
    " 'problem_type_1_category_encoded',\n",
    " 'problem_type_2_category_encoded',\n",
    " 'problem_type_3_category_encoded',\n",
    " 'problem_type_4_category_encoded',\n",
    " 'problem_type_1_per_usage1_category_encoded',\n",
    " 'problem_type_2_per_usage1_category_encoded',\n",
    " 'problem_type_3_per_usage1_category_encoded',\n",
    " 'problem_type_4_per_usage1_category_encoded',\n",
    " 'problem_type_1_per_usage2_category_encoded',\n",
    " 'problem_type_2_per_usage2_category_encoded',\n",
    " 'problem_type_3_per_usage2_category_encoded',\n",
    " 'problem_type_4_per_usage2_category_encoded',\n",
    " 'fault_code_type_1_count_category_encoded',\n",
    " 'fault_code_type_2_count_category_encoded',\n",
    " 'fault_code_type_3_count_category_encoded',\n",
    " 'fault_code_type_4_count_category_encoded',\n",
    " 'fault_code_type_1_count_per_usage1_category_encoded',\n",
    " 'fault_code_type_2_count_per_usage1_category_encoded',\n",
    " 'fault_code_type_3_count_per_usage1_category_encoded',\n",
    " 'fault_code_type_4_count_per_usage1_category_encoded',\n",
    " 'fault_code_type_1_count_per_usage2_category_encoded',\n",
    " 'fault_code_type_2_count_per_usage2_category_encoded',\n",
    " 'fault_code_type_3_count_per_usage2_category_encoded',\n",
    " 'fault_code_type_4_count_per_usage2_category_encoded',\n",
    " 'cat1_encoded',\n",
    " 'cat2_encoded',\n",
    " 'cat3_encoded',\n",
    " 'cat4_encoded',     \n",
    " 'pca_1_warn',\n",
    " 'pca_2_warn',\n",
    " 'pca_3_warn',\n",
    " 'pca_4_warn',\n",
    " 'pca_5_warn',\n",
    " 'pca_6_warn',\n",
    " 'pca_7_warn',\n",
    " 'pca_8_warn',\n",
    " 'pca_9_warn',\n",
    " 'pca_10_warn',\n",
    " 'pca_11_warn',\n",
    " 'pca_12_warn',\n",
    " 'pca_13_warn',\n",
    " 'pca_14_warn',\n",
    " 'pca_15_warn',\n",
    " 'pca_16_warn',\n",
    " 'pca_17_warn',\n",
    " 'pca_18_warn',\n",
    " 'pca_19_warn',\n",
    " 'pca_20_warn',\n",
    " 'pca_roll_features'\n",
    "]\n",
    "\n",
    "label_var = ['label']\n",
    "key_cols =['key','deviceid','date']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Assemble features\n",
    "va = VectorAssembler(inputCols=(input_features), outputCol='features')\n",
    "df = va.transform(df).select('deviceid','date','label','features')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set maxCategories so features with > 10 distinct values are treated as continuous.\n",
    "featureIndexer = VectorIndexer(inputCol=\"features\", \n",
    "                               outputCol=\"indexedFeatures\", \n",
    "                               maxCategories=10).fit(df)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remember to do “StringIndexer” on the label column, fit on the entire dataset to include all labels in index. Also, the label column has to be Double instead of Integer type.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Fit on whole dataset to include all labels in index.\n",
    "labelIndexer = StringIndexer(inputCol=\"label\", outputCol=\"indexedLabel\").fit(df)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare train and test dataset using time split method \n",
    "-  training data: year 2012-2014\n",
    "-  testing data: year 2015"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1431846\n",
      "655155\n"
     ]
    }
   ],
   "source": [
    "training = df.filter(df.date > \"2011-12-31\").filter(df.date < \"2015-01-01\")\n",
    "testing = df.filter(df.date > \"2014-12-31\")\n",
    "\n",
    "print(training.count())\n",
    "print(testing.count())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-------+\n",
      "|label|  count|\n",
      "+-----+-------+\n",
      "|  0.0|2054739|\n",
      "|  1.0|  30363|\n",
      "+-----+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## show the distribution of label \"0\" and \"1\"\n",
    "df.groupby('label').count().show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Down-Sample Negative examples:\n",
    "-  This is a ***highly umbalanced data*** with way more label \"0\" than \"1\" (\"1\" only accounts for 1.5%).\n",
    "-  So we need to down sample the negatives while keeping all positive samples.\n",
    "-  To make label \"1\" to \"0\" ratio close to 1:10 (you can use other ratio for example 1:5), we need to down-sample the \"0\"s (take 13.5% of all the label \"0\"s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+------+\n",
      "|label| count|\n",
      "+-----+------+\n",
      "|  0.0|190823|\n",
      "|  1.0| 20777|\n",
      "+-----+------+\n",
      "\n",
      "+-----+------+\n",
      "|label| count|\n",
      "+-----+------+\n",
      "|  0.0|645547|\n",
      "|  1.0|  9608|\n",
      "+-----+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# SampleBy returns a stratified sample without replacement based on the fraction given on each stratum\n",
    "train_downsampled = training.sampleBy('label', fractions={0.0: 0.135, 1.0: 1.0}, seed=123).cache()\n",
    "train_downsampled.groupby('label').count().show()\n",
    "\n",
    "testing.groupby('label').count().show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cache results when necessary especially if your downstream work (e.g. recursive modeling) use that data over and over again. Here in our case, after the train and test datasets are prepared, we cache them in memory. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1899\n",
      "1899\n"
     ]
    }
   ],
   "source": [
    "# cache datasets in memory\n",
    "train_downsampled.cache()\n",
    "testing.cache()\n",
    "\n",
    "# check the number of devices in training and testing data\n",
    "print(train_downsampled.select('deviceid').distinct().count())\n",
    "print(testing.select('deviceid').distinct().count())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set model storage directory path. This is where models will be saved.\n",
    "modelDir = \"/mnt/resource/PysparkExample/Outputs/\"; "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Binary classification models:\n",
    "-  Random Forest classifier\n",
    "-  Gradient-Boosted Tree"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------+----------+------+\n",
      "|indexedLabel|prediction| count|\n",
      "+------------+----------+------+\n",
      "|         1.0|       1.0|   475|\n",
      "|         0.0|       1.0|  1792|\n",
      "|         1.0|       0.0|  9133|\n",
      "|         0.0|       0.0|643755|\n",
      "+------------+----------+------+\n",
      "\n",
      "CPU times: user 17.9 ms, sys: 6.75 ms, total: 24.6 ms\n",
      "Wall time: 28.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Train a RandomForest model.\n",
    "rf = RandomForestClassifier(labelCol=\"indexedLabel\", featuresCol=\"indexedFeatures\", numTrees=100)\n",
    "\n",
    "# Chain indexers and forest in a Pipeline\n",
    "pipeline_rf = Pipeline(stages=[labelIndexer, featureIndexer, rf])\n",
    "\n",
    "# Train model.  This also runs the indexers.\n",
    "model_rf = pipeline_rf.fit(train_downsampled)\n",
    "\n",
    "# Save model\n",
    "datestamp = unicode(datetime.datetime.now()).replace(' ','').replace(':','_');\n",
    "rf_fileName = \"RandomForest_\" + datestamp;\n",
    "rfDirfilename = modelDir + rf_fileName;\n",
    "model_rf.save(rfDirfilename)\n",
    "\n",
    "# Make predictions.\n",
    "predictions_rf = model_rf.transform(testing)\n",
    "predictions_rf.groupby('indexedLabel', 'prediction').count().show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('deviceid', 'string'),\n",
       " ('date', 'timestamp'),\n",
       " ('label', 'double'),\n",
       " ('features', 'vector'),\n",
       " ('indexedLabel', 'double'),\n",
       " ('indexedFeatures', 'vector'),\n",
       " ('rawPrediction', 'vector'),\n",
       " ('probability', 'vector'),\n",
       " ('prediction', 'double')]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions_rf.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Area under ROC = 0.59777\n",
      "Area under PR = 0.130851\n",
      "\n",
      "Accuracy = 0.983325\n",
      "Weighted Precision = 0.974624\n",
      "Weighted Recall = 0.983325\n",
      "F1 = 0.978217\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAFMCAYAAABLWT5cAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd4FNXbxvFvKkkIgZCEXkM59N5R6fxQQRDpVUQsiF1B\nEbGLiogFFUSQoggCAhYQEERAqvR66CUkkARSCKmbnfePDbwBKSFkM1uez3VxsWV25pkE7j1nyjke\nhmEghBDi9niaXYAQQjgjCU8hhMgFCU8hhMgFCU8hhMgFCU8hhMgFCU8hhMgFb7MLEK5NKWUARwFL\n1kvewN/AM1rrS1nLlATGAXdlLZcKTNZaT862Hl9gLNAD8Mj6Mx94S2udfp3t3tbyQtwuaXmK/NBa\na11Na10NqAkUBUYDKKUKYgvTU8DlZboBjyml3si2jtlAPaCZ1loBzYC6wHc32ObtLi/EbZGWp8hX\nWus0pdQfwANZLz0MRGutx2Zb5oRSajCwSSn1KVAGuA8or7WOz1rmglLqEaD2tdtQStW81fJKqTXA\nt1rr7699ntVaHp1V2y+An9b66azlQoGTQCmgNPA1UBJIA4Zorf/Ni5+TcHzS8hT5SikVDPQDNmS9\n1Ar47drltNZ7gGigSdYym7TWF65ZJlprveo6m7nd5a/HI6vFOh/oku31LsAq4CKwGJilta4KPAEs\nUUpJg8RNSHiK/LBGKXVQKXUMOI4tfD7Meq8oEHODz53Ler9o1uOcut3lr+c3AK31FsBDKVU36/UH\ngZ+AakAxYHrWcv9g248Wd7hd4STkW1Lkh9Za64isLu8hYJ7W+vIJpFhsXeDrKY6t9RmMrYucU7G3\nufz1ZG+1LgQeUEodwXZSqz+27n8AcEApdXm5ICDkDrcrnIS0PEW+0VrHAp8DH2V7eRm2E0RXUUrV\nwtaC3AKsAZoppUpds0wRpdTbSimPaz6ek+UzAa9sbwffpPQF2I7R/g/4W2t9EYgEEi+fCMv6U0pr\nvegm6xEuRMJT5LcJQAulVKus598D3kqpCUopHwClVDlgJvCO1vqS1vogMA+Yq5QqnrVMUeBHIFRr\nfdXQYDlcPgrb2XeUUs2BqjepeSO2VvDD2LrsYDtpFKGU6pG1jlCl1I9ZVw8INyDhKfJVVqvtA+Bj\npZSH1joT6ICtlXlQKXUQ2xnur7XWH2f76DDgL2Bd1jJ/Zz0fcYNN3Wr5T4D7lVIHgEHAipvUbGA7\nOdQe+DXba32AEVnrXwusunztqnB9HjKepxBC3D5peQohRC5IeAohRC5IeAohRC5IeAohRC5IeAoh\nRC44zR1GFkumEReXbHYZdhMcHIDsn/Ny5f1z5X0DCAsrdO1NFjniNC1Pb2+vWy/kxGT/nJsr758r\n79udcJrwFEIIRyLhKYQQuSDhKYQQuSDhKYQQuSDhKYQQuSDhKYQQuSDhKYQQuWDXi+SzRgNfAkzU\nWk+65r32wPvYRvReqrV+x561CCFEXrJbyzNrRO0vsE32dT2fAw8BLYGOSqka9qpFCCHymj277WnY\n5s6OvPYNpVQ4cEFrfVprbQWWAu3sWIsQQuQpu3Xbs2ZHtGSbWTC7Elw93Ww0UMletQghREqahX3H\nL3Dy3EUiopOIu5jCxj/nsX3lt7lan6MMDJKjG/PDwgrZuw5Tyf45N1feP2fat8xMK6ejk9h7NJb4\npDRi4lKIS0xl77HzZFistmUsGexe/iln9D+Ac4VnJLbW52WluU73/loxMRftVpDZwsIKyf45MVfe\nP2fYt5Q0C4dOx7PtUAzrd0ddd5nQwn7UqxJK1VL+vP/6U5zR/9C8ectcb9OU8NRan1BKBSmlKgAR\nQGegvxm1CCGcT0qahY37znI8KpEzMZc4dS4Ja9Zklj7enpQKLUjzmiXw9fGkevlgggJ88S9gi7sP\nPniH9evW8L//3cs338zIdQ12C0+lVENsc3RXADKy5rf+BTiutV4EPIltHm2AeVrrQ/aqRQjhGs7E\nXmL19gg27D1LWnomAF6eHoSXCqJa+SLUKF+UymUK4+1143Phzz33MoGBQTz++HB8fHxyXYszTT1s\nOHrX4U44Q9foTsj+OS+z9y0tPZOlm06yYe9ZziemAlA0qACt65WmQdUwigX73zQsAY4fP8bYsa/y\n+edfExxc9Kr3cjsYsqOcMBJCiKtcSs1g3a4olm46SVJKBgEFvKlXOZSWtUtSr0oIXp45u9Jy3769\n9O79INHR51i27Hf69RuYJ/VJeAohHIrVMPhndxTz1xwlKSUDD6Bl7RL0a1/1ynHLnNqyZTP9+/ck\nISGe99//KM+CEyQ8hRAO5NDpeOb8eYhT55Io4OPFQ63Cuat2SQoHFrjtda1du4aBA3uTnp7Ol19+\nQ8+effK0VglPIYTpDp6MY/6aIxyPsh1bbVazOD1aVaJokF+u11m2bDnCworz3nsf8r//3ZtXpV4h\n4SmEME16RiZz/jzE2l22azMrlQqiT7sqVCpdONfrXL9+LS1b3k3FiuFs2PAvvr6+eVXuVWRIOiGE\nKS4kpvLBD9tZuyuKUqEFealPPV4b1CjXwWkYBp99NoHu3Tszfvw4ALsFJ0jLUwiRz5JTM9h/Io7v\nVx4i8VI6LWuVYFAnhc8dTHFsGAZvvfU6X331OWXKlOWhh3rmYcXXJ+EphMgXicnp/PbPCf7acYZM\nq4Gnhwd921WhfaMyeHjk6lJLACwWCy+99Cxz5symSpWq/PTTYkqXLpOHlV+fhKcQwq5OnE1k+ZbT\n/HswmkyrgY+3Jx0alaVJjWJUKBF0x+vftu1f5s79gbp16/PjjwsJDQ3Ng6pvTcJTCJGnklMz2Hv8\nAsciE9l6MJq4i2kAFA/2p0390jSrVYKggDs/FmkYBh4eHjRt2oxZs36kefOWFCp052GcUxKeQog8\nYcm0snHvWRb8fZSLyRkA+Pp40lCF0apeKWqUL4qnZ+6759lduHCeQYP68swzz9Ox47107Jj3lyLd\nioSnEOKOJKdaWL09gtXbI4hPSsfX25POLSpQs0Iw5YoXuu27gm4lKiqSXr26ofVBVq5cYUpwgoSn\nEOIO7DgUw/crDxF3MQ0/Xy86Ni5Lh0ZlCSmc+4vbb+bYsSP07NmN06dP8fjjT/HWW+/ZZTs5IeEp\nhLgtKWkW/j0YzcZ9Zzl4Kh5vLw+6tKhAp6bl8ryVmV1ExGm6dOlETEw0r776Os8999IdnaW/UxKe\nQogcSUhKY+O+c8xfc4TLI1kWK+LP8AdrUa64/afpKFWqNPfc05omTZoxZMijdt/erUh4CiFuKNNq\nZYeOZuOuM6zaFkF61hxANSsWpU+7KpQMCcDTzq2/v/5aRZUqVSlTpixffTXV1NZmdhKeQojrio5P\nYfyc7ZxPtF1qFOjvwwN3VaRlrRK5GuUoN37+eT4jRjxOtWo1+PPPtXjmcAzP/CDhKYT4j38PRvPj\nqsPEXUzj7nqlaVgllCplCtv1mOa1pk+fyquvvkShQkG8//54hwpOkPAUQmRjGAaL1h3jtw0nAbi3\naTmG96qfr9NwGIbBxInj+eCDdwkNDWPevEXUrl0n37afUxKeQgjOxSWzdONJdh6J5WKybcqL53rV\npfIdDA2XW5cuJTF//lzKli3H/PmLCQ+vnO815ISEpxBuyjAMTkcnsWZnJOt3R2LJNCgc6EvzmsXp\nenc4xYr452s9FosFwzAIDCzE/PlL8PLyomTJUvlaw+2Q8BTCzVxMTmf30fMsXnfsysmg0MJ+PHh3\nOE1rFM+zWyhvR2pqKo8//gj+/n589dW3lClTNt9ruF0SnkK4AavV4O+dZ/h900niEtO4POF4+eKF\nqBVelAdaVrij8TTvRFLSRQYN6sv69Wu5++5WpKamEhAQYEott0PCUwgXdyYmibmrDrPvRBwAhQN9\nadugDI1UGCVDCppa2/nz5+nbtzs7d+7gvvu6MHnyNPz87HNrZ16T8BTChR06Hc/4H3eQaTWoVq4I\nA/+nTA/My6xWK717P8ju3Tvp23cAEyZ8jre380SS81QqhMgxq2Gw6t8I5q46jAHc26wcPVpVcpi7\ncwA8PT0ZNWo0Gzdu4PXX33Ko2nJCwlMIF5KQlMayzafYeTiW6PgUCvp5M6hTNRpXK2Z2aVfs2bOL\ngwcP0LNnHzp06ESHDp3MLilXJDyFcAGGYbBgzVFWbY8gPcOKj7cnzWuW4KFW4Xc093le27RpA/37\n9yI1NYVmzVpQtmw5s0vKNQlPIZxcfFIa3684xPZDMXgAHRuXpUvLChT08zG7tKusXPkHQ4cOIjMz\nky+//MapgxMkPIVwWmcvJLNk/fErE6uVL1GI53rUybdBO27HggXzeOaZJ/H19WXGjDm0bdve7JLu\nmISnEE4mLT2TReuOsWLraQA8PTzo36EqreqVwtvLsQbPuGzPnt0ULBjIDz/Mp0mTpmaXkyc8jMuj\nmjo+Iz8HJ8hvYWGF8nXwhfwm+3dnDMNg+6FYNu8/y55jF0jLyMTTw4P7mpfngZYV7Bqaud03wzCI\niYmhWLFiGIZBRMRph+yqh4UVytVpfml5CuHA4i6msWnfWfYcO8/BU/EAFPTzpk39ctzXvDyB/o51\nXPMyq9XK66+/wpIli/j11+VUrBjukMF5JyQ8hXAwVqvB3uPn+W3jSY5EJFx5PdDfhye61qRauWBT\n7j/PqYyMDJ577inmz59LtWrV8ffP3wFG8ouEpxAO4siZBFZvi2Br1gmgyyqVCqJlnZK0rFUSH2/H\nPKZ5WUpKCo899jDLly+jYcPGzJkzn+DgomaXZRcSnkKYbO/x8yzdePJKtxygdFhB+rSrQvXywXaf\nIygvvffemyxfvoxWrdrw3Xc/EBgYaHZJdiPhKYRJouNTmLfqMDsOxwJQsWQh2jYoQ+NqxfD1MWeE\nozv14ouj8PUtwKhRr1GggONdMpWXJDyFyGex8SksWnecjfvOArZWZq82lakdHmJyZbkTEXGaiRM/\n5v33PyI4uChjx75tdkn5QsJTiHwSEZ3E8i2n2LT/HJlW26jtXVpUoE390k43KMZlhw8fomfPrkRG\nnqFZs+b07NnH7JLyjYSnEHaWabWydmcks1ccAsDL04OHWoXTsXFZ0wYgzgu7du2gT5/unD9/njFj\n3nKr4AQJTyHswjAMjkUmsnzLKXYfPU+6xYqnhwf3NivHvU3LE+Dn3P/1NmxYz4ABvbl0KYkJEz5n\n4MCHzS4p3zn3b1AIBxR1/hK//nOCTfvPAeDn68VddUrSvmEZyhUvZHJ1ecPX1xdvby+mTp3BAw88\naHY5ppDwFCKPZFqtLFhzlJVbI7AaBn6+XgzrXIP6VcPMLi3P7N+/jxo1atKoURP+/XcPQUH5PzWx\no7BreCqlJgLNAAN4Vmu9Ndt7TwEDgEzgX631c/asRQh7OhmVyKSfdnLwVDwFfL0Y1FHRQIVRwEkv\nObqeb775ijFjXuHjjz9j0KAhbh2cAHa7XUEp1QqoorVuDgwFPs/2XhDwMnC31vouoIZSqpm9ahHC\nno6cSWDEx39x8FQ8NSoEM/7JFjSvVcJlgtMwDD788D3GjHmF4sVL0Lixa4yKdKfs2fJsBywG0Fof\nUEoFK6WCtNaJQHrWn0ClVBIQAFywYy1C5LmTZy/y144I1u6KAqCRCuOJrrUc+r7z22W1WnnmmWeY\nNGkS5ctXYP78JVSoUNHsshyCPcOzBLAt2/OYrNcStdapSqm3gGNACjBXa33IjrUIkSeshsH+ExdY\nvO44xyITASgW7M+Ae6tTq1wRk6vLe6tXr2TSpElUr16Tn35aRPHiJcwuyWHk5wmjK1/HWd320UBV\nIBFYrZSqq7XedbMVhIW5xpnKG5H9c2x7j8byxU87iYy9dOW1lwc0pGWdUng56CDEd6pv3x4kJX1D\njx49CA4ONrsch2LP8IzE1tK8rBQQlfW4OnBMax0LoJRaBzQEbhqeMpiu83Lm/cu0Wpn++8Ert1MG\nBfhwb7PytKpXCj9fby5cuOTU+3etxMQEhg8fxsiRo6lTpx7Dhg0jJuaiy+zftXL7pW7P8FwBvAVM\nUUo1ACK11pd/+ieA6kopf611CtAIWGrHWoTIlUupGYz5djMJSemUDi3Iw/dWo1Jp1z3LHB0dTZ8+\n3dm7dzclS5Zm/Ph6ZpfksOwWnlrrDUqpbUqpDYAVeEop9TCQoLVepJQaD/yllLIAG7TW6+xVixC5\nkZpu4YuFe0hISsfXx5NR/Rs47MjteeH06VP07NmVY8eOMnDgED744GOzS3JoMoeRg3Clbt/1ONP+\nZVqtbN5/jnmrj3AxOYPq5YN5vlfdm84T5Ez7dz0nThyna9d7iYqK5NlnX2T06LFXBitx9n27FZnD\nSIg8cC4umS8W7iEy9hIeHlArvCjDu9Vy2Fkp80rx4iWoWDGcxx4bzlNPPWN2OU5BwlMIID4pjfl/\nHWXTvrMYQImiATzXsw7FggPMLs2utmzZTLVq1QgKKsyCBb/g7S2RkFPykxJub8n64/y24cSVeYMG\ndqxKayceYzOnli79jccfH0KTJs1ZsGCJBOdtkp+WcFvnLiTz/cpD7Dt+gcKBvjSvUYL2jcpQNMjP\n7NLsbu7cH3juuafw8/PnmWeed/kvCnuQ8BRuJyY+he+WHrgy4Vp4qSAe61LD5bvol02ePImxY0dT\npEgRfvxxIQ0bNja7JKck4SncypYD55j5hyYlzULVMoVpXL04bRu4fhf9sgsXzvP5559QokRJfvpp\nMdWqVTe7JKcl4SncgmEYLFl/nF/+OYEH0KddFTo0KuM2oWm1WvHw8KBo0RDmzVtMUFAQ5ctXMLss\npybhKdzCnJWHWbU9Al9vT17oXY+qZV1vEI8bSU9P5+mnH6d8+YqMHj2W2rXrmF2SS5DwFC7NMAx+\n/ecEq7ZHAPD6w40pHVrQ5KryT3JyMkOHDmTVqpU0adKMtLQ0l59PPb9IeAqXZcm08tuGE/zyzwkK\nBfjwaOcabhWcCQnx9O/fiy1bNtGuXQemTZstwZmHJDyFS4pNSOH1aVtIS8/E28uTMYMaEVbE3+yy\n8k1GRgbdu3dhz55ddO/eg88/n4yvr6/ZZbkUCU/hco6eSWDqb/tJS88kqKAvz/Ws41bBCeDj48OA\nAYM5eHA/48Z9jKena99eagYJT+FSVm+P4PsVtkkJOrcoz4N3h7vNGXWAAwf2ExMTzT33tGbIkEfN\nLselSXgKl3HkTMKV4OzfoSrtGpYxuaL8tW3bVvr2fYj09Ay2bNlFsWLFzC7JpUl4CqdnGAYrt55m\n4dpjAAy9vzota5c0uar89ffffzF4cD9SU1P49NMvJTjzgYSncGoJl9L5aM52os4n4+XpQftGZWhe\n070mKfv11yU8+eRQAKZP/5777utsckXuQcJTOC3DMPhy0R6izicTElSAZ3vUpUyxQLPLyne//LII\nHx9fZs36kbvvbmV2OW5DwlM4pZQ0C/NWH+ZIRAKFC/ry/mPN8fF2rzPKSUlJBAYGMmnSFI4dO0r1\n6jXMLsmtuNe/NuES9Kk4Rn69gbW7ogguVICX+9Z3q+A0DIN3332Te+9ty4UL5ylQoIAEpwmk5Smc\nhmEYrNsdxfcrDmHJtNK0RnEGd1L4+brPP+PMzExGjnyB2bO/Izy8EsnJyRQtGmJ2WW7Jff7VCaeW\nlp7JlF/2sfNILADd7q7IAy0rmlxV/kpPT2f48GH88ssiatWqw9y5P8tZdRPlKDyVUiFARa31v0op\nT6211c51CQFAhsXK2l2R/Lz2GClpFsKK+DGsc00ql3HdudNv5NVXX+aXXxbRrFkLvv9+HkFB7vcz\ncCS3DE+lVF/gbSANqAV8oZTarrWeZu/ihHv792A0M/84yKVUCwDNahZnQIeqBPi57tzpN/P0089h\nsWQwbtzHBAS4x6j3jiwnR9lfAOoCMVnPXwIes1tFQgB/bD7FV4v3cinVQpPqxXhtYEMe61LT7YLz\n3LmzjB8/DqvVSoUKFfnss68kOB1ETrrtCVrrZKUUAFrrFKVUun3LEu4qw2Llm1/3sU3bvquf7FaL\nxtXc87jeiRPH6dmzKydPnqB69Zp07vyA2SWJbHISnrFKqcGAv1KqAdCb/2+FCpFnklMzeOmrDaSm\nZwLw5pDGlCteyOSqzLF//z56936Qc+fO8uKLo7j//i5mlySukZNu+xNAY6AQ8C3gDwy1Z1HC/RyP\nSmT01M1XgvPdR5u6bXBu3bqZrl3v5dy5s7z77geMGvWaW40M5Sxy0vLspLUekf0FpdQTwGT7lCTc\nzdJNJ1mw5igATaoXY+j91fHx9jK5KvNER0eTmprCF19MpnfvfmaXI27ghuGplKoPNABeUkplP0Lt\nA4xFwlPcoZQ0CzOWHWTrwWh8vD0Zcm81mrnZoB7ZRUaeoVSp0tx/fxe2bNlFyZKlzC5J3MTNuu2p\nQHGgCHB3tj9NgJftX5pwZWkZmXw8dwdbD0bj5+vFW480cevgnD17Bk2a1GXFimUAEpxO4IYtT631\nAeCAUmq11npT9veUUg/ZvTLhsqLjkpmx7CDHoy7i5+vFh080p1CA+86v8/nnE3n33TcICQmheHH3\n/QJxNjk55hmplPoICM16XgBoCyy0W1XCJWVYrPyx+SRLN58iLT2TSqWDeKlPfQr4uOfxTcMwePvt\nsXz55WeUKlWa+fOXUKVKVbPLEjmUk/CcDSwDugCTgK7AQHsWJVzP7+uPMXvZgSt3Cw3upLinbim3\nPou8aNECvvzyMypVqsz8+UsoU6as2SWJ25CT8LRorT9QSnXSWn+plJoG/Aj8aefahAuwZFqZs/IQ\na3ZGAtCqXinubVqOYsFyl0y3bg9x5MhhhgwZRlhYmNnliNuUk+s8/ZVSZQCrUiocyAAq2LUq4RJO\nnr3IqMkbWbMzkgA/b14b1JDBnaq5dXAmJSUxYsTjnD59Ck9PT0aOHC3B6aRy0vL8CGgHjAd2ApnA\nHHsWJZybJdPKz38f448tpwBoWqM4T/euT3qKe9/VGxd3gX79erBt278EBQXx/vvjzS5J3IFbhqfW\nevHlx0qpokAhrXWcXasSTmv30Vi+XrKPtKw7hVrXK8WgTtUoHFiAGDcOz7Nno+jVqxsHDx6gR4/e\nvPXW+2aXJO7QzS6S9wSGYRuGboPW+kettUUplaaU+lJr/VS+VSkcntUwmLPyEKu3nwGgcKAvbz/S\nxK0vQbrsxInj9OjxAKdOneTRRx/n3Xc/xNPTfaYNcVU3a3l+ARQFNgJPKKVCgX3AN8CifKhNOImY\n+BRmLDvIgZNxBPr70KN1Je6pKxd5XxYYWAhfX19efvlVXnrpFbe+wsCV3Cw862mtWwJknWE/CZwA\nemutt+VDbcIJXEhM5Z2Z/5KUkkGxIv6MGdyIQH/3GnPzRvbt20uVKlUJDQ1lxYq/CQx0v2mRXdnN\n+g5XDlBprS8BGmgqwSkui01I4b3Z20hKyaB+lVDGPd5MgjPL6tUrue++djz/vG1MHQlO13Ozlqdx\nzfM0rXWmPYsRzmPH4RimLNlHusVKQxXG8G61pDuaZfHihTz11GN4eXnRteuDZpcj7ORm4VlKKfVI\ntuclsz/XWk+3X1nCkZ27kMyXP+/FwwN6tqlEpyblJDizzJw5nZEjnycwsBDffz+P5s1bml2SsJOb\nhedGbKMoXbYp23MDuGV4KqUmAs2yln9Wa70123tlsd2p5Ats11o/cXulCzNkWKx8PHcHVsOgfYMy\n3Nu0vNklOYyoqEjGjn2VkJAQ5s1bRO3adc0uSdjRzUZVGnInK1ZKtQKqaK2bK6WqYwvb5tkWmQBM\n0FovUkp9qZQqp7U+dSfbFPYVEZPE14v3cj4xjdKhBenXQQaxANsAH2AbRm7GjDmUK1eOSpWqmFyV\nsDd7XmzWDlgMV4a3C1ZKBcGVa0jvBn7Jev8pCU7Htk1H89GcHUSdT6Z6+WBG9W9gdkkOwWKx8Pzz\nI/j2228BaNOmnQSnm8jJ7Zm5VQLIfmY+Juu1RCAMuAhMzJpUbp3W+tVbrTAszLXntHHU/Vuz7TRf\nLd6LYcCg+6rTs13uWpyOun+5lZqaSr9+g1m0aBFHjx7i4Ycfxtvbnv+lzONqv7u8kJ+/aY9rHpcG\nPsN27ejvSqn7tda/32wFMTEX7VedycLCCjnk/q3ZcYbZyzV+Bbx5rEsN6lYOzVWdjrp/uZWUdJHB\ng/uxbt3f3HXXPSxd+htxcSlml2UXrva7u1Zuvxhu2W1XStVVSv2rlDqY9fx1pVTTHKw7EltL87JS\nQFTW41jgpNb6aNblT6uAmrdXurC3PzafYtZyTWCAD6P61adu5dBbf8gNJCcn89BDXVi37m86dbqf\nOXMWUKiQtMzcTU6OeU4CHuH/g28e8EkOPrcC6AGQ1TWP1FpfBNBaW4BjSqnLB4caYrsIXzgAwzBY\nvO4YP/11hOBCBXilfwO3nQb4evz9/WnWrCV9+vRn+vTZ+Pn5mV2SMEFOuu0ZWuvdSikAtNaHlFKW\nW31Ia71BKbVNKbUBsAJPKaUeBhK01ouA54AZWSeP9gC/5nYnRN4xDIN5q4+wYutpQgv78XLf+oQV\n8Te7LIdw7NgRUlPTqFGjJm+++S6GYcgAH24sRyPJK6UqknXHkVLqXq4+fnlDWutXrnlpV7b3jgB3\n5bBOkQ+sVoNZyzVrd0VSMiSAl/rUJ7hQAbPLcgh79uymd+8H8fT05J9/tlK4cBG5McDN5SQ8XwSW\nAEoplYDtBM8gexYl8t+FxFTm/HmY7YdiKFc8kBd61yNIhpMDYNOmDfTv34ukpIt88MEEChcuYnZJ\nwgHkJDzTtdZ1lFJh2O5vT7R3USL/JF5K57cNJ1izMxJLppXKpQvzXM86BPjJAB8AK1f+wdChg7BY\nLHz99bd0797T7JKEg8hJeP6qlIoHvsd2O6VwEbuPxjLp571YMq2EFvajS8sKtKhVAi85jgfYjv9+\n8cWnAMya9SPt2//P5IqEI8nJNBxVlVINgV7ABqWUBmZrrefZvTphFxmWTP7YcprFa49hAI1UGI89\nUBNvLwnNyywWC97e3sycOYejR4/QqFETs0sSDiZH/1u01tu01qOw3VJ5Ettc7sIJJaVk8OGcHSxa\newwPDw9DWwf0AAAgAElEQVRG9q3P8AdrS3BmMQyDTz75iN69HyQ1NZXg4KISnOK6btnyVEqVBB4C\nemK7rXIuUMPOdQk7sGRambJkL8ciEyno5817w5oRVFBOCl1mtVp5443RTJnyFeXKlSc2NoYyZcqa\nXZZwUDk55vkvtgvjX9Ra/2vneoSdWA2D6UsPsO9EHJVLF2Zkv/rS2szm8gAf8+bNQalq/PTTYkqW\nlHmYxI3dbPbMklrrKKANYMl6Lfzy+1rrY/YvT+SF2PgUvsuaoK1S6SBe7FNPgvMaL774DPPmzaFB\ng4bMmbOAokVDzC5JOLibtTwnAP2A5dgukM9+RbABhF/vQ8JxGIbBNh3DN7/ux5JppU6lEB7tXIMC\nPl5ml+ZwBg9+hPj4eL788huZb0jkyM0GQ+6X9fC+rPE4r1BKNb/OR4QDOXchmelLD3A4IgGA/h2q\n0rZBabkrJpvY2FiWL19K//6DaNCgETNnzjG7JOFEbtZtLwKEANOVUv34/5anDzATkGHEHVRk7CXG\n/7iDhEvp1K8SSte7KsrAHtc4cyaCXr26cfjwIcqUKUurVm3MLkk4mZt125sDzwP1gNXZXrdi68oL\nB3QmJonxc3eSeCmdfu2r0L6RnC2+1pEjh+nZsytnzkQwfPgz3HNPa7NLEk7oZt32ZcAypdQTWuvJ\n+ViTyKWI6CTGz93BxeQMBnSsStsGZcwuyeHs3r2TPn26Exsby5gxb/L008/LoQyRKzfrtg/RWn8H\nlFZKvX3t+1rrsXatTNyWo5EJfDZ/N0kpGQzqpGhdr7TZJTmkrVs3c/78ecaP/5TBgx+59QeEuIGb\nddutWX/fcuxOYa7th2KY9vsBUtMsPHxvNe6pK9cnXisxMYGgoMIMHfo4LVrcTfXqcp+HuDM3vNhP\naz0z6++3gE+y/p6M7fjnO/lTnriVAycuMOnnPaSkWRjQsaoE53UsWDCPRo1qs3PndgAJTpEncjKH\n0RdAL6VUUWADMAL42t6FiVtLuJTOj6uOADC4k6KNHOP8j2+/nczw4cMwDEhPzzC7HOFCcnKbSX2t\n9TRsoyrN0Fr3BirbtyxxK1sPRjNm6iYiYpKoUSGYVnKM8yqGYTB+/DhGjx5JsWLFWbx4KU2a5GTe\nQiFyJif3tl8+FdkZGJP1WOZmMNHq7RF8v+IQAHfXKcngTtVMrsjxzJkzm/Hjx1GuXAXmz19MxYpy\nQ5zIWzkJz0NKqf1AjNZ6p1JqEHDBznWJG1i3O/JKcA69vzota5c0uSLH1L17T7Zv38bLL79CiRLy\nMxJ5Lyfd9kex3ePeIev5PmQOI1PsPBLLjKUHAXi0swTntVJSUnjjjddISIjH39+fCRM+k+AUdpOT\nlqc/0AV4WyllAJuAT+1alfiPxOR0Pl+wGw/gmR51qFc51OySHMrFi4kMHNiHDRvW4+Xlxdix/7k0\nWYg8lZOW51QgCJiS9bh41t8in2RarUxZsg+AVvVLS3BeIzY2lgcf7MyGDet54IEHGTXqNbNLEm4g\nJy3P4lrrvtme/6aUWmOnesQ1DMNgzsrDHDgZR51KIfTvUMXskhxKRMRpevXqxpEjhxk48GE++mgi\nXl4y5J6wv5y0PAsqpQIuP1FKFQT87FeSyG7pppP8teMMZcICefyBmjKz5TXS0lKJj4/jmWde4OOP\nP5PgFPkmJy3PKcBBpdTlKTgaAq/bryRx2Yqtp1n49zE8gGd71MG/QE5+Xe7h5MkTlCtXnkqVqvD3\n35sJCwszuyThZm7ZjNFaTwdaYhvDcwbQQms9y851uTXDMNi47yxzVx3G08ODVwc0JKSwNPYvW79+\nLa1bt2DcONtdwhKcwgw3bcoope4DqgHrtdZL8qck93YhMZXJv+zjSEQCHh4wonttKpcpbHZZDmPZ\nst957LGHsVqt1K5dx+xyhBu7YctTKfUm8BpQCpiqlOqfX0W5qx2HYnhj+haORCRQNKgAbw9tSr0q\ncmb9snnz5vDIIwPw8vLmhx/m06VLN7NLEm7sZi3P/wF3a60tSqnCwELgh/wpy/0sXH2YGb/vx8MD\nBnasSuv6Mt9QdseOHeHZZ4cTFBTEnDkLaNSoidklCTd3s/BM1VpbALTWCUopOY1pJ79vPMHCv20z\nOffvUFVGR7qO8PDKfPrpl9StW1+GlBMO4WbhadziucgDv/5znEXrjhMW7M+zD9WhVGhBs0tyGFar\nlTfeGE2bNu1p27Y9ffrIkSPhOG4WnjWUUrNu9FxrLfe336GES+ksXn8cgLeGNcdPLuG8IiMjg6ef\nfoKff57Pv/9upU2bdnIYQziUm4XnqGuer7JnIe4m4VI6E+ftxDCgd9vKlC1eiJiYi2aX5RCSk5MZ\nNmwwK1cup1GjJsyZM1+CUzicm82eOTM/C3EncRfT+HDOdqLjUmhdvzQdZHrgK5KSkujXrwebNm2g\nTZt2TJ/+PQULyqEM4Xiko5jPMq1Wvlq0h+i4FDo1KcfAjlXx9JRW1WX+/v4UK1acbt26M3v2PAlO\n4bDkfr98tmV/NEcjE6lXOZSebSpJdzTL6dOn8Pb2pmTJUnz11VS8vLzkPnXh0HLU8lRKhSilGmU9\nltbqHdiwNwqAXm0rS3Bm0fognTt3pFevbiQnJ+Pr6yvBKRxeTmbP7IttAOQZWS99oZQaas+iXFXc\nxTT2n4ijcunClCgacOsPuIEdO7bRtWsnoqIi6dt3IAEB8nMRziEnrcgXgLpATNbzl4DH7FaRC1uz\n4wwG0KJWCbNLcQjr1v1N9+5diI+P59NPv2T48KfNLkmIHMtJeCZorZMvP9FapwDp9ivJNaVnZLJ8\n6yl8fTxpXL2Y2eWYLjMzk9deG0lGRjrffjuLfv0Gml2SELclJyeMYpVSgwF/pVQDoDf/3woVObRq\newTpGVburlOSgn4+ZpdjKsMw8PLyYvbseZw6dZK7725ldklC3LactDyfABoDhYBvsU0I92hOVq6U\nmqiU2qiU2qCUanyDZca5+rQeicnpLNt0CoD6Vd177Mmvv57Es88Ox2q1Ur58BQlO4bRu2fLUWscD\nI253xUqpVkAVrXVzpVR1YDrQ/JplagD3ABm3u35nkWGxMnbaFpJSMmhZq4TbTt5mGAbjxr3NxIkf\nU6JESc6dO0vJkqXMLkuIXLtleCqlTnOdQUG01uVu8dF2wOKsZQ8opYKVUkFa68Rsy0zANmbomzmu\n2IlYrQaTl+wl8VI6QQE+PHJ/dbNLMkVmZibDhw9n8uTJVKwYzvz5SyQ4hdPLyTHPu7I99sUWiv45\n+FwJYFu25zFZryUCKKUeBv4GTuRgXU4n02plxrKD7DgcS4miAYwZ1Mhtr+t8/vkRzJ37AzVr1mbe\nvEUUKyYnzITzy0m3/eQ1Lx1WSi0HJt7mtq4kh1KqKDAEaA+UzukKwsIK3eYmzZGUnM5b327i4Mk4\nvL08GPfUXYQWufX3jbPs3+3q3bsHUVERLF68mCJFiphdjt246u8PXHvfcisn3fa217xUFqiUg3VH\nYmtpXlYKiMp63BYIA9YBBYBKSqmJWuvnb7ZCZxh1KCI6iS8X7eFcXApBAT68MaQJRobllrWHhbnW\nqEoJCfFs3bqZ9u3/R8uW7ejatSuxsUkutY/ZudrvLztX3jfI/RdDTrrt2acZNrB1u5/IwedWAG8B\nU7IucYrUWl8E0FovABYAKKUqADNuFZzOICEpjbHTtwBQvkQhXh/UyC0H/Th37hx9+nTn4MH9/P77\nSho0cN9DFsJ15SQ8X9Rab7/dFWutNyiltimlNgBW4Kms45wJWutFt7s+R2cYBlN/2w9Ag6phjOhe\n2+SKzHHy5Al69uzKiRPHGTLkUerVa2B2SULYRU7C82Ns3ezbprV+5ZqXdl1nmRNA69ys31FkWq1M\n//0A+0/EUSTQlye71TS7JFMcPHiAXr26cfZsFC+88DKjRo2RFqdwWTkJz1NZF7FvItttmVrrsfYq\nyplYMq2M/mYTsQmpBAX48NrARnh5uufAUwsWzOPs2SjeeWccjz/+lNnlCGFXOQnP41l/xDWshsH0\npQeITUiloJ83bw1tSuGCvmaXle/S09Px9fVl9OixtG3bnhYt7rr1h4RwcjcMT6VUf631D1rrt/Kz\nIGfyw8pDbNp3jkB/H9551D2D89dfF/P222P5+effKFu2nASncBs361/KmJ03se/4Bf7afgaAMYMa\numVw/vDDLIYNe5jY2FjOnIkwuxwh8pV7Hpy7Q3uPn2fCvJ0A9GpTmWLB7jeA76RJn/H88yMoUqQI\nixb9RrNmLcwuSYh8dbNjni2UUqeu87oHYOTg3naXdDE5nc8X7AGgY+OydGrqfj+G7777lrfffp1S\npUrz00+LqVpVmV2SEPnuZuG5A+iTX4U4g4SkNMb9sB1LppXKZQrTp10Vs0syRZcu3fjrrz95772P\nKFvW/b48hICbh2fqde5rd2uTfrZNGazKFuHlfvXNLidfpaWlMWXKVzzxxFOEhoYya9Zcs0sSwlQ3\nC88t+VaFE9h3/AJHIxPx8fbkpb718HSji7+TkpIYMqQ/f//9FxZLBi+8MNLskoQw3Q3DU2s9Kj8L\ncWRWq8H3KzQAT3St6VYXwcfFXaBfv55s27aVjh078eSTMkmbECBn22/JMAwWrTvGubgUalYsSv0q\n7jONxtmzUXTrdh/btm3loYd68d13P+Dvn5OhXIVwfRKet7Bo3TF+33iSQH8fBv7Pvc4qnz59ihMn\njvPoo4/z5Zff4OPj3hPXCZFdTm7PdFtrd0WydKPtaq0XetelWA4GNHYFFy6cp2jREBo3bsrq1esJ\nD68sA3wIcQ1ped7ANh3NjGUHsRoGo/rVp0KJILNLyhdbtmymWbP6zJ49A4BKlapIcApxHRKe1xEd\nn8KUX/YBMKxzDVS5YJMryh+rV/9Jr15duXjxIr6+7ne7qRC3Q7rt10hKyWD8nO1YMg3uqVuK5rVK\n3PpDLmDJkp8ZPnwYnp6efPfdD3TqdJ/ZJQnh0CQ8rzF31WHOJ6YRFODD4E7ucYJo7949PPbYEAoW\nDGT27Lm0bHm32SUJ4fAkPLM5dDqezfvP4efrxYdPtHCbY321atVm1KjXaNeuA3XrutedU0Lklhzz\nzJKcauGLhbvJtBr0a1+VAr5eZpdkV4Zh8MEH77J3r22QkxdeGCnBKcRtkJYntiCZvvQAl1ItNKpW\njLvqlDS7JLvKzMzkpZee5YcfZrFp0wYWLfrdbVrZQuQVCU9g3e4oth+KISTIj2Gdq5tdjl2lpaXx\n5JOP8ttvS6hTpx7ffjtLglOIXHD78Dx17iKzl9vuW3/6odr4eLtudz0pKYmHH+7P2rV/0aLFXcye\nPZdChdzj+lUh8prbH/Oc+Ycm02rQp21lyhUvZHY5duXh4UFKSjKdOt3Hjz8ulOAU4g64dctz34kL\nHI9KBKB947ImV2M/Z89GERgYSGBgIX78cQEBAQXx9nbrX70Qd8xtW56WTCuz/7B111/s7brjcx47\ndpTOnTsyaFBfMjIyCAoqLMEpRB5w2/BcvO440fEpVCodRM2KRc0uxy727t1Dly7/49Spk7RsebeE\nphB5yC3/N8XEp7B000kKB/rydPc6ZpdjF5s3b6J//54kJiYwbtx4hg593OyShHApbheeVsPgi4W2\nC8MfvDucIBecbz0tLY0nnniES5eS+OqrqfTo0dvskoRwOW4Xnut3RxERk0RQgA93u+jF8AUKFGDa\ntFnExsbQseO9ZpcjhEtyq/C0GgZL1h8H4OmH6rjcxeEzZkwjNjaGl156hQYNGpldjhAuza1OGO04\nFEPcxTSqlClMpdKFzS4nzxiGwaeffszIkc8zffo3nD9/3uyShHB5btPyNAyDeauP4AEMcqG5iAzD\n4I03XmPy5EmUKVOW+fMXExISYnZZQrg8twnP7YdiiU1IpXyJQpQOCzS7nDxhGAbPPz+COXNmU6VK\nVebPX0KpUqXNLksIt+AW3fZMq5W5qw4DMLCj67Q6PTw8qFGjJvXq1eeXX5ZLcAqRj9wiPNftiuJ8\nYip1KoUQXsr57+dOSrrIjh3bAHjsseH89ttK6aoLkc9cPjythsGs5Rpfb0+XONZ5/vx5HnqoC927\nd+Hw4UMAMlmbECZw+fDcf/wCACGF/Sga5GdyNXcmMvIMXbt2YseO7XTp0pWKFcPNLkkIt+Xy4fnH\nllMAdL+nksmV3Jljx47QuXNHDh3SPPHECD799Eu5V10IE7n0/74LiansPxFHUEFfGlQNNbucO/L5\n5xOJiDjN6NFjefbZF13uAn8hnI1Lh+fWg9EAtK5XymnDxjAMPDw8+OCDCXTo0In77+9idklCCFy8\n277lwDk8PTxo27CM2aXkysqVf3Dffe1JSIjHz89PglMIB+Ky4XnibCLHoy5So2IwQQHOdzZ6wYJ5\nDB7cj/3793LgwH6zyxFCXMNlw3PFltMAtHfCVue0aVMYPnwYAQEFmTdvMc2atTC7JCHENex6zFMp\nNRFoBhjAs1rrrdneawOMAzIBDTyqtbbmxXbjLqbxr46meLA/tcOd6+Lxb7+dzOjRIwkLK8a8eYuo\nVau22SUJIa7Dbi1PpVQroIrWujkwFPj8mkW+AXporVsChYBOebXtpZtOYsk0+F+Tck53oqht2/Y0\nbNiIX39dLsEphAOzZ7e9HbAYQGt9AAhWSmW/N7Kh1joi63EMkCdNRKvVYN3uSABa1CqRF6u0O4vF\nwsyZMzEMg/DwyixduorwcOe+LlUIV2fP8CyBLRQvi8l6DQCtdSKAUqok0BFYmhcb3XLwHOkZVqqX\nD8bXxysvVmlXqampPPLIAB5++GGmT58K4HStZSHcUX5e5/mfRFBKFQN+BYZrrW85gm9YWKFbbmTT\n/t0A9O9UPUfLmykxMZFevXqxZs0a2rVrx4gRjxMY6BrD5V2Po/8+7pQr758r71tu2TM8I8nW0gRK\nAVGXn2R14ZcBr2mtV+RkhTExF2/6fnR8CruPxFK+RCFKFilwy+XNFBsbS9++D7Fr1w7uv/8BFi78\nicTEdFJSHLfmOxEWVsihfx93ypX3z5X3DXL/xWDPbvsKoAeAUqoBEKm1zv4bmABM1Fr/kVcbXJ51\nH3vbBqUdvuu7bdtWdu/eSb9+A5k6dQYFChQwuyQhxG3wMAzDbitXSn0A3ANYgaeA+kACsByIAzZm\nW3yO1vqbm6zOuNm3X1pGJs98to4Mi5VJz91DgJ9j3nmakpKCv78/AFu3bqZRoyZ4eHi4xbe77J9z\ncuV9AwgLK5SrlpZdE0Zr/co1L+3K9jhPm1q7jsSSYbFyX7PyDhucu3fvZODAPkyc+AVt23agceOm\nZpckhMglx0yZXFiz4wwATaoXM7mS69uwYT0DBvTm0qUkoqKibv0BIYRDc4nwjLuYxsFT8QQV9KVs\nMcc7W718+TKGDRtMZmYm33zzHV27dje7JCHEHXKJ8Pxrh+1a+wdaVnC4E0Vbt27m4Yf7UaBAAWbM\nmEPbtu3NLkkIkQdcIjx3HI4FoGHVMJMr+a8GDRrRr98g+vTpJ8c4hXAhTj+q0pmYJM7EXKJWxaIU\nDnSMy30Mw2Dy5ElERp7By8uLCRM+k+AUwsU4fXj+s+csAA0cpNVptVp57bWRjB07mpdffs7scoQQ\nduLU3XarYbBx31n8C3jTvKb5g4BkZGTw7LPDWbBgHtWr1+CTT74wuyQhhJ04dctTn4on4VI6DaqE\nUsDX3EFAUlJSGDKkPwsWzKNRoyYsWbKM4sXND3QhhH04dXiu3m47y35XnZImVwJJSUkcPnyI1q3b\nMn/+EooUCTa7JCGEHTlttz3DYmWbjqFE0QCqli1iWh0XLpynUKEgwsLCWLJkGcHBReU+dSHcgNO2\nPLcfsg0VWrNiUdOu7YyIOM3993fg2WeHY7VaKVGipASnEG7C6cOzWc3ipmz/8OFDdO7ckaNHj1Cq\nlOOP4iSEyFtO2W23Wg32Hr9ASJAf4SWDbv2BPLZz53b69n2I8+fPM3bsO4wY8Wy+1yCEMJdThueZ\n2EukpFloWDUs31t8SUlJ9OvXg7i4OD755AsGDBicr9sXQjgGpwzPIxHxAFQuUzjftx0YGMiECV9g\nsWTQpUu3fN++EMIxOGV4HjxlC8/8PMs+d+4PeHh40Lt3P+699/58264QwjE5XXharQb7jl8guFAB\nigf758s2p0z5ktdff5XQ0DDuv78LgYEyGZYQ7s7pwjPy/CWS0yzUqxJq9+OdhmHw4Yfv8skn4ylR\noiQ//bRYglMIAThheB49kwBApdL2Pd5ptVoZPfplpk+fSoUKFZk/fwnly1ew6zaFEM7D6a7zPB5l\nm4jK3pcoeXh44OXlRY0atfj11xUSnEKIqzhdy/NMbBKeHh6UCi1ol/UnJydz7txZKlYM5513PiA5\n+ZJ01YUQ/+FULU/DMIiKTaZYsD8+3nlfekJCPL17P0jXrvcSGXkGT09PCU4hxHU5VXjGXUwjOc1C\nmbC8b3VGR0fTrdv9bN68kWbNmhMa6hiDKwshHJNThefJs7bjnWXyeIbMU6dO0qVLR/bt28PgwUP5\n+utp+Pr65uk2hBCuxanC8/DlM+2l8vZM+5gxr3D8+DGef/4lPvroE7y8zB1YWQjh+JzqhNGJqEQ8\ngPBSeXumfeLESXTqdB/9+g3M0/UKIVyXU7U8oy4kUzTID/8Cd575a9eu4ZFHBpKenk5ISIgEpxDi\ntjhNeCYkpZGQlJ4nlyj9/vuv9OvXgxUrlrFr1448qE4I4W6cptseFXsJgJIhAXe0njlzZvPCC0/j\n7x/AzJlzZD51kSNRUZEMGtQHpaoBtplSw8Mr89JLr+Dl5UVKSgrjx7/P/v178fb2Jjg4hBdfHHVl\nEsDTp0/x+ecTiI+PIzPTSu3adXjqqedMPTGZmZnJqFHP8/zzIylduoxpdSQlJfHWW6+RlJSEv38A\nb775LkFBV5/X+PffLUya9Cmenp50796Dzp27cezYUSZO/AgAT08vRo16jYsXL/L99zN4550P7F63\n07Q8z11IBiCsSO4HA5k27Ruee+4pChcuzMKFv3DPPa3zqDrhDsqVK8+kSd8wadI3TJnyHRZLBitX\n/gHAuHHjCA0N47vv5jB16iwGDBjMiy8+g8ViITMzkzFjRtKv3yCmTp3FtGmzAfjuu6lm7g6LFy+g\nbt36pgYnwE8/zaF+/YZ8/fU0WrVqw/ffz7zqfYvFwscfj+Ojjyby1VffsmXLZgCmT5/CgAEP88UX\nU7j//gf44YeZKFWNkJBQ/vrrT7vX7TQtz7MXbC3PkMJ+uV5H7dp1qVSpMjNmzLnSghAit2rUqEVE\nxGmSky+xbt065sz5+cp7derUo0aNmqxbtwZ//wDKlatA/foNAdutv8OHP4OHx9VtF4vFwrvvvsG5\nc1H4+hZgzJi32Lp1M8eOHWXEiOdITk5m0KDeLFjwK336PEizZi0JDg5m2bLfmTvXtu1ly37jyJFD\n9O07kHHj3sFiycDT05NRo16nRImrp8JesGAeU6Z8B8CKFctYsGAeXl6eVKhQiVGjXmPp0l/ZtGkD\nCQkXGDPmHdauXcOff/6Bh4cnd9/dmr59BxAdfY533hl7pf4xY966Kow3bFjPnDmzrtruAw90p2PH\nTleeb9u2lVdfta2jZct7GDnyuauW1/ogZcqUpVgx25Q7b789DoDChYuQkGC7AufixUQKF7YNUdmj\nR2/ee+9N2rRpn7NfZC45T3jG2lqetzsMXWZmJn///Rdt27anSZOmrFu3BW9vp9ltcR0/rT7C1oPR\nebrOxtWK0att5Rwvb7FYWLfub7p1e4gzZyIIDw//z7+rKlUUp06dxN/fnypVql71XoEC/20ELFv2\nGyEhIbz55nv8+edy1q9fe8MJBS0WC82ataBZsxZs3/4vx44dJTy8EuvW/U3fvgOYOvVr+vTpT+PG\nTdm4cT0zZ37LqFFjrnz+7Nmz+Pr6Xukep6SkMGHCFxQqVIinnhrG0aNHADh37iwLF85n927NmjWr\n+OqraQA8+eRQ2rRpT1zceYYMGUaDBo347bcl/PzzfJ5++vkr22nR4i5atLjrpj/L8+fPX5mqOzg4\nmPPnY696/+zZSHx8fHj99VeIjY2me/dedOjQiUcffYJHHx3EjBlTsVqtTJ1qC+kyZcpy7txZUlNT\n8fPLfWPrVpwmRaLjbOEZWjjn4Zmens6IEY+xePHPfPPNd3Tr9pAEp8i1U6dOMmLEYwAcPXqE/v0H\ncc89rTl8+BCZmZn/Wd4wDDw9vQAPrFbrLdev9UEaNWoMQPv2/wNg6dJfb7h8jRo1Abjnnjb88886\nSpcuw/HjR6lVqw4ffPAOp06dZObMaVit1ivhdFlsbAxhYcWuPA8KCuLVV18E4OTJ4yQk2AYcr169\nBh4eHhw4sI+IiNM8/fTjACQnX+Ls2UhKlizFp59+zLRpU7h4MRGlqt9yP2/GMIzrvnbu3Fm++moa\naWmpPPLIAJo0acaUKV/y+ONP0bHjvSxcOI8ZM6by9NMvABASEsL587F2PSThNEkSE59CUIBPju9p\nT05O5pFHBrB69Z80bdqcNm3a2blCkV96ta18W63EvHL5mCfAmDEjKVu2PAClS5fm+PHjZGRk4OPj\nc2X5I0cOcc89rfHx8WXhwp+uWld6ejoREacID////fDy8sRqvTo8so9Za7FYrnrP29u2rVat2vD6\n668QHl6Jpk2b4+Hhgbe3D++88yGhoaE33J/L687IyOCTTz5ixow5hISEXtVtvrwNb28fmjdvyciR\nr121jvfff4umTZvRrVsP/vrrTzZsWH/V+znptoeGhnLhQiyBgYHExsb859bookVDqFatBn5+fvj5\n+REeXokzZyLYs2cXTz75NACNGzflww/fu+G+2oPTnDCKjU+haFDOmuDx8XH07NmV1av/pH37jsyb\nt+jK8RAh8sLw4c8yefIXpKamEhBQkDZt2jB9+jdX3t+zZxeHDmmaN7+Lxo2bcu5cFOvXrwVsY8V+\n/QwXTTAAAAu1SURBVPUXrFq18qp1VqtWg+3btwLwzz/rmDVrOgEBBa90Y3fv3nndWkJDbRMh/vnn\nclq3tjUSatSoxbp1awDbMcUVK/74z2eio22HPpKTL+Hl5UVISCjnzp3l4MED/wlqpaqzffs2UlNT\nMQyDTz/9mLS0VOLj4yldugyGYbB+/d9kZGRc9bkWLe66cpLt8p/swQnQpEkzVq+2neBZs2YVTZs2\nv+r9mjVrc+TIYdLS0khPT+f06dOULFma0qXLsn//XgAOHNhPmTJlr3zmwoULhITc+IsjLzhNeGZY\nrAQXuv7xn2stXfobW7dupnv3nsyc+SMBAXd2eZMQ1ypVqjStW7dj5kzbMcDRo0eTnp7G4MF9GTZs\nELNmTeeddz7Ay8sLT09PJkyYxC+/LGLo0IEMH/4ogYGBDB36+FXrbN/+/9q79+Co6iuA49/EYBQE\nBzExvjo+oMcngiCIAiKxAoJl0IiCiDBgtCwjWto6o7ZawddISEm01scUpxQIKFDBQrFKYaBoEWNG\nxuLREREJtUBRoVOMEugfvxtcluRucpPdvRfOZ4YZsvd1fnd3zt77u7/f2QHs3buXiROLmT9/LoMG\nDaF798sOdhds2bL5sIdMdXr37ktVVSWdO3cBYNy4YlavXkksdgczZ77ARRddfMj6BQUF1NTUsHu3\ne9By2WU9GT9+NDNnvsDIkbdRVjb9kARaUFDA8OEjiMXuoLh4DB06dCA39ziGDr2B0tKnmDz5bgoL\nB1BVVcm6dW836VwWFd2C6kYmTBhPZeW7jBw5GoAZM0rYtq2a3NxcRo8eSyx2B7HYeEaMGEX79u2J\nxSYxZ84sJk4sZvnypYwd67pUqqu3kp+fn9L+ToCs+voYwuj6ya8e6NflNEYPbPgpeW1t7cF56cuW\n/ZkBAwaRnR2N74e8vLbs2LEn02GkjLUvfF5+uYKamm8YNWqM73pRa1tZWQkXXtiZwsIfNWr9vLy2\ngX7PJxqZxdOuTcMDijdu/Cf9+vViw4b3ARg0aHBkEqcxmTBsWBFVVZVUV2/NdCgt5uOPle3btzc6\ncTZHpLJL29b1J8/169cxdOhAVD+ksnJ9mqMyJppycnKYNq0s44PkW1KnTsLUqU+m5ViRedoO0Ob4\nw8NduXIFY8bcSk3NN5SX/46bbx6ZgciMMUebSCXP1rmtDvl77do13HrrTWRnZzNz5mwGDrwuQ5EZ\nY442kbptz211aLiXXNKVPn2uYu7cBZY4jTFpFank2SrHPUmvqJjNnj27adOmDRUVC+ndu2+GIzPG\nHG1SetsuIqXA5cABYJKqvhO37BrgMaAWWKqqU5Ltr9UxWUyZ8hDl5aWsWvU3nn32xVSFbowxvlJ2\n5SkiVwGdVLUXMA4oS1ilDLgRuBK4VkQu8Nvfgf21PDblPsrLSzn33I488MBDKYnbGGMaI5W37YXA\nnwBUdSPQXkTaAYjIOcAuVf1cVfcDS731G1S5tIQF8/9I585dWLx4+SFTsYwxJt1SmTwLgB1xf+/w\nXqtv2XbgVL+d7dyygR49erFo0Wvk5dlvqhtjMiudQ5X8pkAlnR717d7dgaZQRUleXttMh5BS1r7o\nOpLbFlQqrzy38f2VJsBpwL8aWHa695oxxkRCKpPn60ARgIhcCmxT1T0AqroZaCciZ4lIDjDEW98Y\nYyIhpVWVROQJoC+wH4gBXYGvVXWRiPQF6iahLlDVaSkLxBhjWlhkStIZY0yYRGqGkTHGhIUlT2OM\nCSCUVZVaelpnmCRp29XA47i2KTDem0QQGX7ti1vncaCXqvZLc3jNluT9OxOYCxwLVKrqXZmJMrgk\n7YsBo3Cfz/Wqek/9ewkvEbkIeBUoVdWnE5Y1KbeE7sqzpad1hkkj2vY8UKSqVwJtgYFESCPah/d+\nRbKSSyPaVwKUqGoPoFZEfpDuGJvDr33e7MCfA31UtTdwgYhcnplIgxGRNkA58GYDqzQpt4QuedLC\n0zpDpsG2ebqpat1vIuwAOqQ5vuZK1j5wCeaBxA0jwu+zmQ30ARZ7y2OquiVTgQbk9/596/07wRte\n2BrYlZEog6sBrqOeMeVBcksYk2eLTusMGb+2oaq7AUTkVOBa3BsYJb7tE5ExwCpgc1qjajl+7csD\n9gClIrLG65qImgbbp6rfAL8GNgGfAf9Q1Y/SHmEzqOo+Vd3bwOIm55YwJs9EzZrWGXKHxS8i+cAS\nYIKq/if9IbWog+0TkZOAsbgrzyNFVsL/TwdmAFcBXUVkcEaiajnx71874H7gh8DZQE8RuSRTgaVB\n0twSxuR5JE/r9Gtb3Qd0GfCgqkZxxpVf+/rjrs5WA4uAS72HE1Hi176dwGeq+omq1uL61S5Mc3zN\n5de+84FNqrpTVb/FvY/d0hxfKjU5t4QxeR7J0zobbJunBPcU8C+ZCK4F+L13r6jqBap6OTAM9zT6\n3syFGohf+/YBm0Skk7duN9yIiSjx+3xuBs4XkeO9v7sDH6c9whQJkltCOcPoSJ7W2VDbgOXAl8Bb\ncavPUdXn0x5kM/i9d3HrnAW8FNGhSn6fzY7AS7iLkg3ATyI41MyvfXfiul72AWtV9ReZi7TpRKQb\n7gLlLOA7oBr3gO/TILkllMnTGGPCLoy37cYYE3qWPI0xJgBLnsYYE4AlT2OMCcCSpzHGBBDKqkom\n87zhRMqhQ6cA7lHVqga2eRjIUdUHm3HcfriqN+95Lx0HVOIq/HzXxH0NxNULeFRErgC+UNVNIvIb\nYJaqvtuMOB/GDdv51HspB9gK3KmqX/tsdxpwnqquCHpsEw6WPI2fHRkai7mh7rgikgVUAHcCT/tt\nlMibbFA34WAsMA83S6alSqnNiv+iEJEncVMY7/PZ5mrcbB1LnhFnydM0mYicBzyHGyzdDjeddHnc\n8hzgRUBwdSHfU9WYiBwLPAN0xJXcm6uqvnPdVfWAiKwBzvP2PRj4FfA/71+xqlZ7g7v74yrnVAO3\nAyOAa4AFwE1ADxG519t+Kq526iRVXevt+w3cIOoPgN/iKgedANyvqm804tSsBYq9ffXGDbiu8fYz\nATcJ4lEgS0R24b4MmnQ+THhYn6cJogD4paoWAnfjEkK8i4GeqtpLVa8AqkTkRGASbsrf1UBP4BYR\n6ex3IBE5DrgeWC0irXFJ+UZvH8uAqSLSHjcbppeq9gEWAqfU7cOb3VQFTE64XZ7N99MR83FXhK8D\nz+LqcvYHfgy86H0h+MWZA4zk+26Ok3EzjPrjioXcr6qf4mYgzVLV6UHOhwkPu/I0fvJEZGXCazfh\nikU8JSKP4qqmn5ywzkZgp4gsxVWImq+qX3uV8s/wiu6C68/sCLyfsP3FCcddoqrzRKQL8O+4mqcr\ngbtU9UsRWQ6sEpFFwDxV3SoiydpXAfwd+Ckuib6sqrVenG1F5CFvve+AfA4vFHGbd4WZhZvGOAN4\nwlv2BTDNS/4n4q46EzX2fJgQsuRp/NTb5ykic3C3mL/3ftbgtfjlXu3HPl5xiSHAOyJyJe4W9hFV\nfSXJcTfUd1xcF0C8rLrXVLXI604YjEuiNyZrnKp+ISKbRKQHcDMuieLFeYOq7kyyi4N9niKyBFdV\naV/dMtzDoxUiMgT4WT3bN/Z8mBCy23YTxCm4fkFwSSc3fqGIdBeR21W1UlUfAd7F1YFcAwz31skW\nkelenc/G+gjIj/t5i2uAt0XkHBG5V1U/9PoMFwKJtSb3A63q2eds3E9OnBT39D0+zpO9p/PJTAAe\nFpEzvL9PAT4QkWNwV+t15yg+juaeD5NBljxNECXAH7xb5TXALhGJf9DxCVAkImtFZAXwFe72+Bng\nvyLyFvA28JWqNvqnHLwq4OOAed5tfSHwIG6IUFcRWScib+KK9S5I2PyvwHMickPC6wtxfZVz4167\nGxgmIqtx1fyTPhlX1c9xD4jqqmA96W23BNfPeaaI3IOrgzlWRKbQzPNhMsuqKhljTAB25WmMMQFY\n8jTGmAAseRpjTACWPI0xJgBLnsYYE4AlT2OMCcCSpzHGBGDJ0xhjAvg/UI3e8844cEIAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f311f8a5dd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 21s, sys: 664 ms, total: 1min 22s\n",
      "Wall time: 2min 14s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "predictionAndLabels = predictions_rf.select(\"indexedLabel\", \"prediction\").rdd\n",
    "metrics = BinaryClassificationMetrics(predictionAndLabels)\n",
    "print(\"Area under ROC = %g\" % metrics.areaUnderROC)\n",
    "print(\"Area under PR = %g\\n\" % metrics.areaUnderPR)\n",
    "\n",
    "# Select (prediction, true label) and compute test error\n",
    "evaluator = MulticlassClassificationEvaluator(labelCol=\"indexedLabel\", predictionCol=\"prediction\")\n",
    "print(\"Accuracy = %g\" % evaluator.evaluate(predictions_rf, {evaluator.metricName: \"accuracy\"}))\n",
    "print(\"Weighted Precision = %g\" % evaluator.evaluate(predictions_rf, {evaluator.metricName: \"weightedPrecision\"}))\n",
    "print(\"Weighted Recall = %g\" % evaluator.evaluate(predictions_rf, {evaluator.metricName: \"weightedRecall\"}))\n",
    "print(\"F1 = %g\" % evaluator.evaluate(predictions_rf, {evaluator.metricName: \"f1\"}))\n",
    "\n",
    "# PLOT ROC curve after converting predictions to a Pandas dataframe\n",
    "%matplotlib inline\n",
    "predictions_rf_pddf = predictions_rf.select('indexedLabel','probability').toPandas()\n",
    "labels = predictions_rf_pddf[\"indexedLabel\"]\n",
    "prob = []\n",
    "for dv in predictions_rf_pddf[\"probability\"]:\n",
    "    prob.append(dv.values[1])\n",
    "     \n",
    "fpr, tpr, thresholds = roc_curve(labels, prob, pos_label=1.0);\n",
    "roc_auc = auc(fpr, tpr)\n",
    "\n",
    "plt.figure(figsize=(5, 5))\n",
    "plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % roc_auc)\n",
    "plt.plot([0, 1], [0, 1], 'k--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC Curve')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Pyspark MulticlassClassificationEvaluator in version 2.0.2 only gives weighted precision and recall. But we would also like to see the raw precision and recall as well."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision: [ 0.98601138  0.20952801]\n",
      "recall: [ 0.99722406  0.04943797]\n",
      "fscore: [ 0.99158602  0.08      ]\n",
      "support: [645547   9608]\n"
     ]
    }
   ],
   "source": [
    "# Use sklearn\n",
    "rf_result = predictions_rf.select('indexedLabel', 'prediction').toPandas()\n",
    "\n",
    "rf_label = rf_result['indexedLabel'].tolist()\n",
    "rf_prediction = rf_result['prediction'].tolist()\n",
    "\n",
    "precision, recall, fscore, support = score(rf_label, rf_prediction)\n",
    "\n",
    "print('precision: {}'.format(precision))\n",
    "print('recall: {}'.format(recall))\n",
    "print('fscore: {}'.format(fscore))\n",
    "print('support: {}'.format(support))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gradient-Boosted Tree classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 130 ms, sys: 13 ms, total: 143 ms\n",
      "Wall time: 11min 50s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Train a GBT model.\n",
    "gbt = GBTClassifier(labelCol=\"indexedLabel\", featuresCol=\"indexedFeatures\", maxDepth=10, minInstancesPerNode=5, maxIter=50)\n",
    "\n",
    "# Chain indexers and GBT in a Pipeline\n",
    "pipeline_gbt = Pipeline(stages=[labelIndexer, featureIndexer, gbt])\n",
    "\n",
    "# Train model.  This also runs the indexers.\n",
    "model_gbt = pipeline_gbt.fit(train_downsampled)\n",
    "\n",
    "# save model\n",
    "datestamp = unicode(datetime.datetime.now()).replace(' ','').replace(':','_');\n",
    "gbt_fileName = \"GradientBoostedTree_\" + datestamp;\n",
    "gbtDirfilename = modelDir + gbt_fileName;\n",
    "model_gbt.save(gbtDirfilename)\n",
    "\n",
    "# Make predictions.\n",
    "predictions_gbt = model_gbt.transform(testing)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------+----------+------+\n",
      "|indexedLabel|prediction| count|\n",
      "+------------+----------+------+\n",
      "|         1.0|       1.0|   847|\n",
      "|         0.0|       1.0|  6106|\n",
      "|         1.0|       0.0|  8761|\n",
      "|         0.0|       0.0|639441|\n",
      "+------------+----------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# show prediction results\n",
    "predictions_gbt.groupby('indexedLabel', 'prediction').count().show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Area under ROC = 0.554151\n",
      "Area under PR = 0.109647\n",
      "\n",
      "Accuracy = 0.977308\n",
      "Weighted Precision = 0.973804\n",
      "Weighted Recall = 0.977308\n",
      "F1 = 0.975512\n",
      "CPU times: user 49.2 ms, sys: 19.3 ms, total: 68.5 ms\n",
      "Wall time: 41.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "predictionAndLabels = predictions_gbt.select(\"indexedLabel\", \"prediction\").rdd\n",
    "metrics = BinaryClassificationMetrics(predictionAndLabels)\n",
    "print(\"Area under ROC = %g\" % metrics.areaUnderROC)\n",
    "print(\"Area under PR = %g\\n\" % metrics.areaUnderPR)\n",
    "\n",
    "# Select (prediction, true label) and compute test error\n",
    "evaluator = MulticlassClassificationEvaluator(labelCol=\"indexedLabel\", predictionCol=\"prediction\")\n",
    "print(\"Accuracy = %g\" % evaluator.evaluate(predictions_gbt, {evaluator.metricName: \"accuracy\"}))\n",
    "print(\"Weighted Precision = %g\" % evaluator.evaluate(predictions_gbt, {evaluator.metricName: \"weightedPrecision\"}))\n",
    "print(\"Weighted Recall = %g\" % evaluator.evaluate(predictions_gbt, {evaluator.metricName: \"weightedRecall\"}))\n",
    "print(\"F1 = %g\" % evaluator.evaluate(predictions_gbt, {evaluator.metricName: \"f1\"}))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision: [ 0.98648415  0.12181792]\n",
      "recall: [ 0.99054135  0.0881557 ]\n",
      "fscore: [ 0.98850859  0.10228851]\n",
      "support: [645547   9608]\n"
     ]
    }
   ],
   "source": [
    "# Use sklearn to calculate the raw precision and recall\n",
    "\n",
    "gbt_result = predictions_gbt.select('indexedLabel', 'prediction').toPandas()\n",
    "\n",
    "gbt_label = gbt_result['indexedLabel'].tolist()\n",
    "gbt_prediction = gbt_result['prediction'].tolist()\n",
    "\n",
    "precision, recall, fscore, support = score(gbt_label, gbt_prediction)\n",
    "\n",
    "print('precision: {}'.format(precision))\n",
    "print('recall: {}'.format(recall))\n",
    "print('fscore: {}'.format(fscore))\n",
    "print('support: {}'.format(support))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparing results from Random Forest and Gradient Boosted Tree:\n",
    "-  Gradient Boosted Tree gives better recall but worse precision compared with Random Forest. For most of the predictive maintenance use cases, the business cost associated with false positives is usually expensive. There is always a trade-off between precision and recall. We want to achieve higher precision rate (fewer false positives) even though that might compromise the recall rate.\n",
    "-  That is why we decided to go with Random Forest model and further optimized it with hyper-parametre tuning."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyper-parameter tuning\n",
    "**Train a random forest classification model using hyper-parameter tuning and cross-validation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "weightedPrecision on test data = 0.974678\n",
      "CPU times: user 3.68 s, sys: 2.29 s, total: 5.97 s\n",
      "Wall time: 51min\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Train a RandomForest model.\n",
    "rf = RandomForestClassifier(labelCol=\"indexedLabel\", featuresCol=\"indexedFeatures\", \n",
    "                            featureSubsetStrategy=\"auto\", impurity=\"gini\", seed=123)\n",
    "\n",
    "# Chain indexers and forest in a Pipeline\n",
    "pipeline_rf = Pipeline(stages=[labelIndexer, featureIndexer, rf])\n",
    "\n",
    "\n",
    "## Define parameter grid\n",
    "paramGrid = ParamGridBuilder() \\\n",
    "    .addGrid(rf.numTrees, [20, 50, 100]) \\\n",
    "    .addGrid(rf.maxBins, [10, 20]) \\\n",
    "    .addGrid(rf.maxDepth, [3, 5, 7]) \\\n",
    "    .addGrid(rf.minInstancesPerNode, [1, 5, 10]) \\\n",
    "    .build()\n",
    "\n",
    "## Define cross-validation\n",
    "crossval = CrossValidator(estimator=pipeline_rf,\n",
    "                          estimatorParamMaps=paramGrid,\n",
    "                          evaluator=MulticlassClassificationEvaluator(metricName=\"weightedPrecision\"),\n",
    "                          numFolds=3)\n",
    "\n",
    "## Train model using CV\n",
    "cvModel = crossval.fit(train_downsampled)\n",
    "\n",
    "## Predict and evaluate\n",
    "predictions = cvModel.transform(testing)\n",
    "evaluator = MulticlassClassificationEvaluator(labelCol=\"indexedLabel\", predictionCol=\"prediction\", metricName=\"weightedPrecision\")\n",
    "r2 = evaluator.evaluate(predictions)\n",
    "print(\"weightedPrecision on test data = %g\" % r2)\n",
    "\n",
    "## Save the best model\n",
    "fileName = \"CV_RandomForestClassificationModel_\" + datestamp;\n",
    "CVDirfilename = modelDir + fileName;\n",
    "cvModel.bestModel.save(CVDirfilename);\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyper-parameter tuning only improved the model performance a little bit. We will then use that model for future scoring."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Spark - python",
   "language": "python",
   "name": "spark-python"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}