KDDmodels.Rmd

---
title: "KDDmodels"
author: "John Mount"
date: "August 5, 2015"
output: html_document
---



listing_6.1_of_section_6.1.1.R
```{r listing_6.1_of_section_6.1.1.R, tidy=FALSE}
# example 6.1 of section 6.1.1 
# (example 6.1 of section 6.1.1)  : Memorization methods : KDD and KDD Cup 2009 : Getting started with KDD Cup 2009 data 
# Title: Preparing the KDD data for analysis 

d <- read.table('orange_small_train.data.gz',  	# Note: 1 
   header=T,
   sep='\t',
   na.strings=c('NA','')) 	# Note: 2 
churn <- read.table('orange_small_train_churn.labels.txt',
   header=F,sep='\t') 	# Note: 3 
d$churn <- churn$V1 	# Note: 4 
appetency <- read.table('orange_small_train_appetency.labels.txt',
   header=F,sep='\t')
d$appetency <- appetency$V1 	# Note: 5 
upselling <- read.table('orange_small_train_upselling.labels.txt',
   header=F,sep='\t')
d$upselling <- upselling$V1 	# Note: 6 
set.seed(729375) 	# Note: 7 
d$rgroup <- runif(dim(d)[[1]])
dTrainAll <- subset(d,rgroup<=0.9)
dTest <- subset(d,rgroup>0.9) 	# Note: 8 
outcomes=c('churn','appetency','upselling')
vars <- setdiff(colnames(dTrainAll),
   c(outcomes,'rgroup'))
catVars <- vars[sapply(dTrainAll[,vars],class) %in%
   c('factor','character')] 	# Note: 9 
numericVars <- vars[sapply(dTrainAll[,vars],class) %in%
   c('numeric','integer')] 	# Note: 10 
rm(list=c('d','churn','appetency','upselling')) 	# Note: 11 
outcome <- 'churn' 	# Note: 12 
pos <- '1' 	# Note: 13 
useForCal <- rbinom(n=dim(dTrainAll)[[1]],size=1,prob=0.5)>0 	# Note: 14 
dCal <- subset(dTrainAll,useForCal)
dTrain <- subset(dTrainAll,!useForCal)

# Note 1: 
#   Read the file of independent variables. All 
#   data from 
#   https://github.com/WinVector/zmPDSwR/tree/master/KDD2009. 

# Note 2: 
#   Treat both NA and the empty string as missing 
#   data. 

# Note 3: 
#   Read churn dependent variable. 

# Note 4: 
#   Add churn as a new column. 

# Note 5: 
#   Add appetency as a new column. 

# Note 6: 
#   Add upselling as a new column. 

# Note 7: 
#   By setting the seed to the pseudo-random 
#   number generator, we make our work reproducible: 
#   someone redoing it will see the exact same 
#   results. 

# Note 8: 
#   Split data into train and test subsets. 

# Note 9: 
#   Identify which features are categorical 
#   variables. 

# Note 10: 
#   Identify which features are numeric 
#   variables. 

# Note 11: 
#   Remove unneeded objects from workspace. 

# Note 12: 
#   Choose which outcome to model (churn). 

# Note 13: 
#   Choose which outcome is considered 
#   positive. 

# Note 14: 
#   Further split training data into training and 
#   calibration. 
```


listing_6.2_of_section_6.2.1.R
```{r listing_6.2_of_section_6.2.1.R, tidy=FALSE}
# example 6.2 of section 6.2.1 
# (example 6.2 of section 6.2.1)  : Memorization methods : Building single-variable models : Using categorical features 
# Title: Plotting churn grouped by variable 218 levels 

table218 <- table(
   Var218=dTrain[,'Var218'], 	# Note: 1 
   churn=dTrain[,outcome], 	# Note: 2 
   useNA='ifany') 	# Note: 3 
print(table218)
# Note this listing was updated: 10-14-2014 as some of results in the book were
# accidentally from older code.  Will update later listings as we go forward.

# Note 1: 
#   Tabulate levels of Var218. 

# Note 2: 
#   Tabulate levels of churn outcome. 

# Note 3: 
#   Include NA values in tabulation. 
```


listing_6.3_of_section_6.2.1.R
```{r listing_6.3_of_section_6.2.1.R, tidy=FALSE}
# example 6.3 of section 6.2.1 
# (example 6.3 of section 6.2.1)  : Memorization methods : Building single-variable models : Using categorical features 
# Title: Churn rates grouped by variable 218 codes 

print(table218[,2]/(table218[,1]+table218[,2]))
```


listing_6.4_of_section_6.2.1.R
```{r listing_6.4_of_section_6.2.1.R, tidy=FALSE}
# example 6.4 of section 6.2.1 
# (example 6.4 of section 6.2.1)  : Memorization methods : Building single-variable models : Using categorical features 
# Title: Function to build single-variable models for categorical variables 

mkPredC <- function(outCol,varCol,appCol) { 	# Note: 1 
   pPos <- sum(outCol==pos)/length(outCol) 	# Note: 2 
   naTab <- table(as.factor(outCol[is.na(varCol)]))
   pPosWna <- (naTab/sum(naTab))[as.character(pos)] 	# Note: 3 
   vTab <- table(as.factor(outCol),varCol)
   pPosWv <- (vTab[pos,]+1.0e-3*pPos)/(colSums(vTab)+1.0e-3) 	# Note: 4 
   pred <- pPosWv[appCol] 	# Note: 5 
   pred[is.na(appCol)] <- pPosWna 	# Note: 6 
   pred[is.na(pred)] <- pPos 	# Note: 7 
   pred 	# Note: 8 
}

# Note 1: 
#   Given a vector of training outcomes (outCol), 
#   a categorical training variable (varCol), and a 
#   prediction variable (appCol), use outCol and 
#   varCol to build a single-variable model and then 
#   apply the model to appCol to get new 
#   predictions. 

# Note 2: 
#   Get stats on how often outcome is positive 
#   during training. 

# Note 3: 
#   Get stats on how often outcome is positive for 
#   NA values of variable during training. 

# Note 4: 
#   Get stats on how often outcome is positive, 
#   conditioned on levels of training variable. 

# Note 5: 
#   Make predictions by looking up levels of 
#   appCol. 

# Note 6: 
#   Add in predictions for NA levels of 
#   appCol. 

# Note 7: 
#   Add in predictions for levels of appCol that 
#   weren’t known during training. 

# Note 8: 
#   Return vector of predictions. 
```


listing_6.5_of_section_6.2.1.R
```{r listing_6.5_of_section_6.2.1.R, tidy=FALSE}
# example 6.5 of section 6.2.1 
# (example 6.5 of section 6.2.1)  : Memorization methods : Building single-variable models : Using categorical features 
# Title: Applying single-categorical variable models to all of our datasets 

for(v in catVars) {
  pi <- paste('pred',v,sep='')
  dTrain[,pi] <- mkPredC(dTrain[,outcome],dTrain[,v],dTrain[,v])
  dCal[,pi] <- mkPredC(dTrain[,outcome],dTrain[,v],dCal[,v])
  dTest[,pi] <- mkPredC(dTrain[,outcome],dTrain[,v],dTest[,v])
}
```


listing_6.6_of_section_6.2.1.R
```{r listing_6.6_of_section_6.2.1.R, tidy=FALSE}
# example 6.6 of section 6.2.1 
# (example 6.6 of section 6.2.1)  : Memorization methods : Building single-variable models : Using categorical features 
# Title: Scoring categorical variables by AUC 

library('ROCR')

calcAUC <- function(predcol,outcol) {
    perf <- performance(prediction(predcol,outcol==pos),'auc')
    as.numeric(perf@y.values)
 }

for(v in catVars) {
   pi <- paste('pred',v,sep='')
   aucTrain <- calcAUC(dTrain[,pi],dTrain[,outcome])
   if(aucTrain>=0.8) {
      aucCal <- calcAUC(dCal[,pi],dCal[,outcome])
      print(sprintf("%s, trainAUC: %4.3f calibrationAUC: %4.3f",
        pi,aucTrain,aucCal))
   }
 }
```


listing_6.7_of_section_6.2.2.R
```{r listing_6.7_of_section_6.2.2.R, tidy=FALSE}
# example 6.7 of section 6.2.2 
# (example 6.7 of section 6.2.2)  : Memorization methods : Building single-variable models : Using numeric features 
# Title: Scoring numeric variables by AUC 

mkPredN <- function(outCol,varCol,appCol) {
  nval <- length(unique(varCol[!is.na(varCol)]))
  if(nval<=1) {
    pPos <- sum(outCol==pos)/length(outCol)
    return(pPos+numeric(length(appCol)))
  }
  cuts <- unique(as.numeric(quantile(varCol,
                        probs=seq(0, 1, 0.1),na.rm=T)))
  varC <- cut(varCol,cuts)
  appC <- cut(appCol,cuts)
  mkPredC(outCol,varC,appC)
}

for(v in numericVars) {
   pi <- paste('pred',v,sep='')
   dTrain[,pi] <- mkPredN(dTrain[,outcome],dTrain[,v],dTrain[,v])
   dTest[,pi] <- mkPredN(dTrain[,outcome],dTrain[,v],dTest[,v])
   dCal[,pi] <- mkPredN(dTrain[,outcome],dTrain[,v],dCal[,v])
   aucTrain <- calcAUC(dTrain[,pi],dTrain[,outcome])
   if(aucTrain>=0.55) {
      aucCal <- calcAUC(dCal[,pi],dCal[,outcome])
      print(sprintf("%s, trainAUC: %4.3f calibrationAUC: %4.3f",
        pi,aucTrain,aucCal))
   }
 }
```


listing_6.8_of_section_6.2.2.R
```{r listing_6.8_of_section_6.2.2.R, tidy=FALSE}
# example 6.8 of section 6.2.2 
# (example 6.8 of section 6.2.2)  : Memorization methods : Building single-variable models : Using numeric features 
# Title: Plotting variable performance 

library('ggplot2')
ggplot(data=dCal) +
   geom_density(aes(x=predVar126,color=as.factor(churn)))
```


listing_6.9_of_section_6.2.3.R
```{r listing_6.9_of_section_6.2.3.R, tidy=FALSE}
# example 6.9 of section 6.2.3 
# (example 6.9 of section 6.2.3)  : Memorization methods : Building single-variable models : Using cross-validation to estimate effects of overfitting 
# Title: Running a repeated cross-validation experiment 

var <- 'Var217'
aucs <- rep(0,100)
for(rep in 1:length(aucs)) {   	# Note: 1 
   useForCalRep <- rbinom(n=dim(dTrainAll)[[1]],size=1,prob=0.1)>0  	# Note: 2 
   predRep <- mkPredC(dTrainAll[!useForCalRep,outcome],  	# Note: 3 
      dTrainAll[!useForCalRep,var],
      dTrainAll[useForCalRep,var])
   aucs[rep] <- calcAUC(predRep,dTrainAll[useForCalRep,outcome])  	# Note: 4 
 }

mean(aucs)

sd(aucs)

# Note 1: 
#   For 100 iterations... 

# Note 2: 
#   ...select a random subset of about 10% of the training data as hold-out set,... 

# Note 3: 
#   ...use the random 90% of training data to train model and evaluate that model on hold-out 
#   set,... 

# Note 4: 
#   ...calculate resulting model’s AUC using hold-out set; store that value and repeat. 
```


listing_6.10_of_section_6.2.3.R
```{r listing_6.10_of_section_6.2.3.R, tidy=FALSE}
# example 6.10 of section 6.2.3 
# (example 6.10 of section 6.2.3)  : Memorization methods : Building single-variable models : Using cross-validation to estimate effects of overfitting 
# Title: Empirically cross-validating performance 

fCross <- function() {
   useForCalRep <- rbinom(n=dim(dTrainAll)[[1]],size=1,prob=0.1)>0
   predRep <- mkPredC(dTrainAll[!useForCalRep,outcome],
      dTrainAll[!useForCalRep,var],
      dTrainAll[useForCalRep,var])
   calcAUC(predRep,dTrainAll[useForCalRep,outcome])
}

aucs <- replicate(100,fCross())
```


listing_6.11_of_section_6.3.1.R
```{r listing_6.11_of_section_6.3.1.R, tidy=FALSE}
# example 6.11 of section 6.3.1 
# (example 6.11 of section 6.3.1)  : Memorization methods : Building models using many variables : Variable selection 
# Title: Basic variable selection 

#    Each variable we use represents a chance of explaining
# more of the outcome variation (a chance of building a better
# model) but also represents a possible source of noise and
# overfitting. To control this effect, we often preselect
# which subset of variables we’ll use to fit. Variable
# selection can be an important defensive modeling step even
# for types of models that “don’t need it” (as seen with
# decision trees in section 6.3.2).  Listing 6.11 shows a
# hand-rolled variable selection loop where each variable is
# scored according to a deviance inspired score, where a
# variable is scored with a bonus proportional to the change
# in in scaled log likelihood of the training data.  We could
# also try an AIC (Akaike information criterion) by
# subtracting a penalty proportional to the complexity of the
# variable (which in this case is 2^entropy for categorical
# variables and a stand-in of 1 for numeric variables).  The
# score is a bit ad hoc, but tends to work well in selecting
# variables. Notice we’re using performance on the calibration
# set (not the training set) to pick variables. Note that we
# don’t use the test set for calibration; to do so lessens the
# reliability of the test set for model quality confirmation.

logLikelyhood <- function(outCol,predCol) { 	# Note: 1 
  sum(ifelse(outCol==pos,log(predCol),log(1-predCol)))
}

selVars <- c()
minStep <- 5
baseRateCheck <- logLikelyhood(dCal[,outcome],
   sum(dCal[,outcome]==pos)/length(dCal[,outcome]))

for(v in catVars) {  	# Note: 2 
  pi <- paste('pred',v,sep='')
  liCheck <- 2*((logLikelyhood(dCal[,outcome],dCal[,pi]) -
      baseRateCheck))
  if(liCheck>minStep) {
     print(sprintf("%s, calibrationScore: %g",
        pi,liCheck))
     selVars <- c(selVars,pi)
  }
}

for(v in numericVars) { 	# Note: 3 
  pi <- paste('pred',v,sep='')
  liCheck <- 2*((logLikelyhood(dCal[,outcome],dCal[,pi]) -
      baseRateCheck))
  if(liCheck>=minStep) {
     print(sprintf("%s, calibrationScore: %g",
        pi,liCheck))
     selVars <- c(selVars,pi)
  }
}

# Note 1: 
#   Define a convenience function to compute log 
#   likelihood. 

# Note 2: 
#   Run through categorical variables and pick 
#   based on a deviance improvement (related to 
#   difference in log likelihoods; see chapter 
#   3). 

# Note 3: 
#   Run through numeric variables and pick 
#   based on a deviance improvement. 

```

From here below we build multiple variable models using only the dCal
portion of the training set.  This is because we built single variable
models on the dTrain portion, so we would like the next level of
training to see these new synthetic variables as they would perform on
new data (and not how they are constructed).  This avoids some
undesirable bias, and helps gets degrees of freedom and
cross-validation right.  In some situations (categorical variables with
very many levels) this can be an important point.  It was always 
mitigated somewhat in the book as we took care to penalize
variables for complexity in the listing 6.11 variable selection.

listing_6.13_of_section_6.3.2.R
```{r listing_6.13_of_section_6.3.2.R, tidy=FALSE}
# example 6.13 of section 6.3.2 
# (example 6.13 of section 6.3.2)  : Memorization methods : Building models using many variables : Using decision trees 
# Title: Building a bad decision tree 

library('rpart')
fV <- paste(outcome,'>0 ~ ',
   paste(c(catVars,numericVars),collapse=' + '),sep='')
tmodel <- rpart(fV,data=dCal)

print(calcAUC(predict(tmodel,newdata=dTrain),dTrain[,outcome]))

print(calcAUC(predict(tmodel,newdata=dTest),dTest[,outcome]))

print(calcAUC(predict(tmodel,newdata=dCal),dCal[,outcome]))
```


listing_6.14_of_section_6.3.2.R
```{r listing_6.14_of_section_6.3.2.R, tidy=FALSE}
# example 6.14 of section 6.3.2 
# (example 6.14 of section 6.3.2)  : Memorization methods : Building models using many variables : Using decision trees 
# Title: Building another bad decision tree 

tVars <- paste('pred',c(catVars,numericVars),sep='')
fV2 <- paste(outcome,'>0 ~ ',paste(tVars,collapse=' + '),sep='')
tmodel <- rpart(fV2,data=dCal)

print(calcAUC(predict(tmodel,newdata=dTrain),dTrain[,outcome]))

print(calcAUC(predict(tmodel,newdata=dTest),dTest[,outcome]))

print(calcAUC(predict(tmodel,newdata=dCal),dCal[,outcome]))
```


listing_6.15_of_section_6.3.2.R
```{r listing_6.15_of_section_6.3.2.R, tidy=FALSE}
# example 6.15 of section 6.3.2 
# (example 6.15 of section 6.3.2)  : Memorization methods : Building models using many variables : Using decision trees 
# Title: Building yet another bad decision tree 

tmodel <- rpart(fV2,data=dCal,
   control=rpart.control(cp=0.001,minsplit=1000,
      minbucket=1000,maxdepth=5)
)
print(calcAUC(predict(tmodel,newdata=dTrain),dTrain[,outcome]))

print(calcAUC(predict(tmodel,newdata=dTest),dTest[,outcome]))

print(calcAUC(predict(tmodel,newdata=dCal),dCal[,outcome]))
```


listing_6.16_of_section_6.3.2.R
```{r listing_6.16_of_section_6.3.2.R, tidy=FALSE}
# example 6.16 of section 6.3.2 
# (example 6.16 of section 6.3.2)  : Memorization methods : Building models using many variables : Using decision trees 
# Title: Building a better decision tree 

f <- paste(outcome,'>0 ~ ',paste(selVars,collapse=' + '),sep='')

tmodel <- rpart(f,data=dCal,
   control=rpart.control(cp=0.001,minsplit=1000,
      minbucket=1000,maxdepth=5)
 )

print(calcAUC(predict(tmodel,newdata=dTrain),dTrain[,outcome]))

print(calcAUC(predict(tmodel,newdata=dTest),dTest[,outcome]))

print(calcAUC(predict(tmodel,newdata=dCal),dCal[,outcome]))
```


listing_6.17_of_section_6.3.2.R
```{r listing_6.17_of_section_6.3.2.R, tidy=FALSE}
# example 6.17 of section 6.3.2 
# (example 6.17 of section 6.3.2)  : Memorization methods : Building models using many variables : Using decision trees 
# Title: Printing the decision tree 

print(tmodel)
```


listing_6.18_of_section_6.3.2.R
```{r listing_6.18_of_section_6.3.2.R, tidy=FALSE}
# example 6.18 of section 6.3.2 
# (example 6.18 of section 6.3.2)  : Memorization methods : Building models using many variables : Using decision trees 
# Title: Plotting the decision tree 

par(cex=0.7)
plot(tmodel)
text(tmodel)
```


listing_6.19_of_section_6.3.3.R
```{r listing_6.19_of_section_6.3.3.R, tidy=FALSE}
# example 6.19 of section 6.3.3 
# (example 6.19 of section 6.3.3)  : Memorization methods : Building models using many variables : Using nearest neighbor methods 
# Title: Running k-nearest neighbors 

library('class')
nK <- 200
knnTrain <- dCal[,selVars]  	# Note: 1 
knnCl <- dCal[,outcome]==pos 	# Note: 2 
knnPred <- function(df) { 	# Note: 3 
    knnDecision <- knn(knnTrain,df,knnCl,k=nK,prob=T)
    ifelse(knnDecision==TRUE, 	# Note: 4 
       attributes(knnDecision)$prob,
       1-(attributes(knnDecision)$prob))
}

print(calcAUC(knnPred(dTrain[,selVars]),dTrain[,outcome]))

print(calcAUC(knnPred(dCal[,selVars]),dCal[,outcome]))

print(calcAUC(knnPred(dTest[,selVars]),dTest[,outcome]))

# Note 1: 
#   Build a data frame with only the variables we 
#   wish to use for classification. 

# Note 2: 
#   Build a vector with the known training 
#   outcomes. 

# Note 3: 
#   Bind the knn() training function with our data 
#   in a new function. 

# Note 4: 
#   Convert knn’s unfortunate convention of 
#   calculating probability as “proportion of the 
#   votes for the winning class” into the more useful 
#   “calculated probability of being a positive 
#   example.” 
```


listing_6.20_of_section_6.3.3.R
```{r listing_6.20_of_section_6.3.3.R, tidy=FALSE}
# example 6.20 of section 6.3.3 
# (example 6.20 of section 6.3.3)  : Memorization methods : Building models using many variables : Using nearest neighbor methods 
# Title: Platting 200-nearest neighbor performance 

dCal$kpred <- knnPred(dCal[,selVars])
ggplot(data=dCal) +
   geom_density(aes(x=kpred,
      color=as.factor(churn),linetype=as.factor(churn)))
```


listing_6.21_of_section_6.3.3.R
```{r listing_6.21_of_section_6.3.3.R, tidy=FALSE}
# example 6.21 of section 6.3.3 
# (example 6.21 of section 6.3.3)  : Memorization methods : Building models using many variables : Using nearest neighbor methods 
# Title: Plotting the receiver operating characteristic curve 

plotROC <- function(predcol,outcol) {
   perf <- performance(prediction(predcol,outcol==pos),'tpr','fpr')
   pf <- data.frame(
      FalsePositiveRate=perf@x.values[[1]],
      TruePositiveRate=perf@y.values[[1]])
   ggplot() +
      geom_line(data=pf,aes(x=FalsePositiveRate,y=TruePositiveRate)) +
      geom_line(aes(x=c(0,1),y=c(0,1)))
}
print(plotROC(knnPred(dTest[,selVars]),dTest[,outcome]))
```


listing_6.22_of_section_6.3.3.R
```{r listing_6.22_of_section_6.3.3.R, tidy=FALSE}
# example 6.22 of section 6.3.3 
# (example 6.22 of section 6.3.3)  : Memorization methods : Building models using many variables : Using nearest neighbor methods 
# Title: Plotting the performance of a logistic regression model 

gmodel <- glm(as.formula(f),data=dCal,family=binomial(link='logit'))
print(calcAUC(predict(gmodel,newdata=dTrain),dTrain[,outcome]))

print(calcAUC(predict(gmodel,newdata=dTest),dTest[,outcome]))

print(calcAUC(predict(gmodel,newdata=dCal),dCal[,outcome]))
```


listing_6.23_of_section_6.3.4.R
```{r listing_6.23_of_section_6.3.4.R, tidy=FALSE}
# example 6.23 of section 6.3.4 
# (example 6.23 of section 6.3.4)  : Memorization methods : Building models using many variables : Using Naive Bayes 
# Title: Building, applying, and evaluating a Naive Bayes model 

pPos <- sum(dCal[,outcome]==pos)/length(dCal[,outcome])
nBayes <- function(pPos,pf) { 	# Note: 1 
   pNeg <- 1 - pPos
   smoothingEpsilon <- 1.0e-5
   scorePos <- log(pPos + smoothingEpsilon) + 
      rowSums(log(pf/pPos + smoothingEpsilon)) 	# Note: 2 
   scoreNeg <- log(pNeg + smoothingEpsilon) +
      rowSums(log((1-pf)/(1-pPos) + smoothingEpsilon)) 	# Note: 3 
   m <- pmax(scorePos,scoreNeg)
   expScorePos <- exp(scorePos-m)
   expScoreNeg <- exp(scoreNeg-m) 	# Note: 4 
   expScorePos/(expScorePos+expScoreNeg) 	# Note: 5 
}
pVars <- paste('pred',c(numericVars,catVars),sep='')
dTrain$nbpredl <- nBayes(pPos,dTrain[,pVars])
dCal$nbpredl <- nBayes(pPos,dCal[,pVars])
dTest$nbpredl <- nBayes(pPos,dTest[,pVars]) 	# Note: 6 

print(calcAUC(dTrain$nbpredl,dTrain[,outcome]))

print(calcAUC(dCal$nbpredl,dCal[,outcome]))

print(calcAUC(dTest$nbpredl,dTest[,outcome]))
# Note: 7

# Note 1: 
#   Define a function that performs the Naive 
#   Bayes prediction. 

# Note 2: 
#   For each row, compute (with a smoothing term) 
#   the sum of log(P[positive & 
#   evidence_i]/P[positive]) across all columns. This 
#   is equivalent to the log of the product of 
#   P[evidence_i | positive] up to terms that don’t 
#   depend on the positive/negative outcome. 

# Note 3: 
#   For each row, compute (with a smoothing term) 
#   the sum of log(P[negative & 
#   evidence_i]/P[negative]) across all columns. This 
#   is equivalent to the log of the product of 
#   P[evidence_i | negative] up to terms that don’t 
#   depend on the positive/negative outcome. 

# Note 4: 
#   Exponentiate to turn sums back into products, 
#   but make sure we don’t cause a floating point 
#   overflow in doing so. 

# Note 5: 
#   Use the fact that the predicted positive 
#   probability plus the predicted negative 
#   probability should sum to 1.0 to find and 
#   eliminate Z. Return the correctly scaled predicted 
#   odds of being positive as our forecast. 

# Note 6: 
#   Apply the function to make the predictions. 

# Note 7: 
#   Calculate the AUCs. Notice the 
#   overfit—fantastic performance on the training 
#   set that isn’t repeated on the calibration or test 
#   sets. 
```


listing_6.24_of_section_6.3.4.R
```{r listing_6.24_of_section_6.3.4.R, tidy=FALSE}
# example 6.24 of section 6.3.4 
# (example 6.24 of section 6.3.4)  : Memorization methods : Building models using many variables : Using Naive Bayes 
# Title: Using a Naive Bayes package 

library('e1071')
lVars <- c(catVars,numericVars)
ff <- paste('as.factor(',outcome,'>0) ~ ',
   paste(lVars,collapse=' + '),sep='')
nbmodel <- naiveBayes(as.formula(ff),data=dCal)
dTrain$nbpred <- predict(nbmodel,newdata=dTrain,type='raw')[,'TRUE']
dCal$nbpred <- predict(nbmodel,newdata=dCal,type='raw')[,'TRUE']
dTest$nbpred <- predict(nbmodel,newdata=dTest,type='raw')[,'TRUE']

calcAUC(dTrain$nbpred,dTrain[,outcome])

calcAUC(dCal$nbpred,dCal[,outcome])

calcAUC(dTest$nbpred,dTest[,outcome])
```

gbm example
```{r gbmexample, tidy=FALSE}
library('gbm')

gmodel <- gbm(as.formula(f),
              distribution='bernoulli',
              n.trees=100,
              interaction.depth=3,
              cv.folds=5,
              data=dCal[,c(outcome,selVars)])
# If we don't subset down data to just the columns we are using then
# cv.folds=5 crashes with "Error in object$var.levels[[i]] : subscript out of bounds".
# So always subset data frames going into gbm().
nTrees = gbm.perf(gmodel)

calcAUC(predict(gmodel,n.trees=nTrees,newdata=dTrain,type='response'),
   dTrain[,outcome])

calcAUC(predict(gmodel,n.trees=nTrees,newdata=dCal,type='response'),
   dCal[,outcome])

calcAUC(predict(gmodel,n.trees=nTrees,newdata=dTest,type='response'),
   dTest[,outcome])
```