Skip to content

Commit

Permalink
Added some regression tests for other bootstrap estimates
Browse files Browse the repository at this point in the history
  • Loading branch information
@topepo
topepo committed Jul 10, 2017
1 parent 9fa556d commit d7ecb35
Show file tree
Hide file tree
Showing 2 changed files with 90 additions and 14 deletions.
53 changes: 53 additions & 0 deletions RegressionTests/Code/gbm.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ cctrl3 <- trainControl(method = "oob")
cctrl4 <- trainControl(method = "none",
classProbs = TRUE, summaryFunction = twoClassSummary)
cctrlR <- trainControl(method = "cv", number = 3, returnResamp = "all", search = "random")
cctrlB632 <- trainControl(method = "boot632", number = 10, search = "random", timingSamps = 11, classProbs = TRUE)
cctrlBopt <- trainControl(method = "optimism_boot", number = 10, search = "random", savePredictions = "final", classProbs = TRUE)
cctrlAdapt <- trainControl(method = "adaptive_boot", number = 15, search = "random", classProbs = TRUE)


set.seed(849)
test_class_cv_model <- train(trainX, trainY,
Expand Down Expand Up @@ -94,6 +98,33 @@ test_class_none_model <- train(trainX, trainY,
test_class_none_pred <- predict(test_class_none_model, testing[, -ncol(testing)])
test_class_none_prob <- predict(test_class_none_model, testing[, -ncol(testing)], type = "prob")

set.seed(849)
test_class_b632_model <- train(trainX, trainY,
method = "gbm",
trControl = cctrlB632,
preProc = c("center", "scale"),
tuneGrid = gbmGrid,
verbose = FALSE)


set.seed(849)
test_class_bopt_model <- train(trainX, trainY,
method = "gbm",
trControl = cctrlBopt,
preProc = c("center", "scale"),
tuneGrid = gbmGrid,
verbose = FALSE)


set.seed(849)
test_class_adapt_model <- train(trainX, trainY,
method = "gbm",
trControl = cctrlAdapt,
metric = "ROC",
preProc = c("center", "scale"),
tuneGrid = gbmGrid,
verbose = FALSE)

set.seed(849)
test_class_rec <- train(recipe = rec_cls,
data = training,
Expand All @@ -106,6 +137,28 @@ test_class_pred_rec <- predict(test_class_rec, testing[, -ncol(testing)])
test_class_prob_rec <- predict(test_class_rec, testing[, -ncol(testing)],
type = "prob")

set.seed(849)
test_class_b632_rec_model <- train(recipe = rec_cls,
data = training,
method = "gbm",
trControl = cctrlB632,
tuneGrid = gbmGrid,
verbose = FALSE)
if(!isTRUE(all.equal(test_class_b632_rec_model$results,
test_class_b632_model$results)))
stop("x/y and recipe interface have different results for B632")

set.seed(849)
test_class_bopt_rec_model <- train(recipe = rec_cls,
data = training,
method = "gbm",
trControl = cctrlBopt,
tuneGrid = gbmGrid,
verbose = FALSE)
if(!isTRUE(all.equal(test_class_bopt_rec_model$results,
test_class_bopt_model$results)))
stop("x/y and recipe interface have different results for B optim")

test_levels <- levels(test_class_cv_model)
if(!all(levels(trainY) %in% test_levels))
cat("wrong levels")
Expand Down
51 changes: 37 additions & 14 deletions RegressionTests/Code/svmRadial.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ cctrl2 <- trainControl(method = "LOOCV")
cctrl3 <- trainControl(method = "none",
classProbs = TRUE, summaryFunction = twoClassSummary)
cctrlR <- trainControl(method = "cv", number = 3, returnResamp = "all", search = "random")
cctrlB632 <- trainControl(method = "boot632", number = 10, search = "random", timingSamps = 11)
cctrlBopt <- trainControl(method = "optimism_boot", number = 10, search = "random", savePredictions = "final")
cctrlAdapt <- trainControl(method = "adaptive_boot", number = 15, search = "random")
cctrlB632 <- trainControl(method = "boot632", number = 10, search = "random", timingSamps = 11, classProbs = TRUE)
cctrlBopt <- trainControl(method = "optimism_boot", number = 10, search = "random", savePredictions = "final", classProbs = TRUE)
cctrlAdapt <- trainControl(method = "adaptive_boot", number = 15, search = "random", classProbs = TRUE)

set.seed(849)
test_class_cv_model <- train(trainX, trainY,
Expand Down Expand Up @@ -86,19 +86,42 @@ test_class_bopt_model <- train(trainX, trainY,
tuneLength = 4,
preProc = c("center", "scale"))

set.seed(849)
test_class_adapt_model <- train(trainX, trainY,
method = "svmRadial",
trControl = cctrlAdapt,
tuneLength = 4,
preProc = c("center", "scale"))

set.seed(849)
test_class_rec <- train(recipe = rec_cls,
data = training,
method = "svmRadial",
trControl = cctrl1)

set.seed(849)
test_class_b632_rec_model <- train(recipe = rec_cls,
data = training,
method = "svmRadial",
trControl = cctrlB632,
tuneLength = 4)

if(!isTRUE(all.equal(test_class_b632_rec_model$results,
test_class_b632_rec_model$results)))
stop("x/y and recipe interface have different results for B632")


set.seed(849)
test_class_bopt_rec_model <- train(recipe = rec_cls,
data = training,
method = "svmRadial",
trControl = cctrlBopt,
tuneLength = 4)
if(!isTRUE(all.equal(test_class_bopt_rec_model$results,
test_class_bopt_rec_model$results)))
stop("x/y and recipe interface have different results for B optim")

# set.seed(849)
# test_class_adapt_model <- train(recipe = rec_cls,
# data = training,
# method = "svmRadial",
# trControl = cctrlAdapt,
# tuneLength = 4,
# preProc = c("center", "scale"))

test_class_pred_rec <- predict(test_class_rec, testing[, -ncol(testing)])

test_levels <- levels(test_class_cv_model)
Expand Down Expand Up @@ -191,10 +214,10 @@ test_reg_bopt <- train(trainX, trainY,

set.seed(849)
test_reg_adapt <- train(trainX, trainY,
method = "svmRadial",
trControl = rctrlAdapt,
tuneLength = 4,
preProc = c("center", "scale"))
method = "svmRadial",
trControl = rctrlAdapt,
tuneLength = 4,
preProc = c("center", "scale"))

set.seed(849)
test_reg_rec <- train(recipe = rec_reg,
Expand Down

0 comments on commit d7ecb35

Please sign in to comment.