Merge pull request #2 from jakob-r/fix1_oneTuningParam

Fix Issue #1
jakob-r · Sep 20, 2017 · 81509be · 81509be
2 parents cf3f3b3 + 0980d6b
commit 81509be
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 30 deletions.
diff --git a/R/calcOuterPerformances.R b/R/calcOuterPerformances.R
@@ -78,7 +78,7 @@ calcOuterPerformance.ResampleResult = function(tuning.resampled, out.res.i, task
   } else {
     y.change = seq_row(par.settings)
   }
-  par.settings = par.settings[y.change, ]
+  par.settings = par.settings[y.change, , drop = FALSE]
 
   par.settings = dfRowsToList(df = par.settings, par.set = par.set)
   par.settings = lapply(par.settings, function(x) trafoValue(par.set, x))

diff --git a/R/plot.R b/R/plot.R
@@ -9,6 +9,8 @@ plot.SimulateOuterPerformanceResult = function(outer.performance, resample.overf
   measure.vars = unlist(outer.performance[c("y.inner.name", "y.outer.name", "cum.y.inner.name", "cum.y.outer.name", "sim.y.outer.name")])
   summary.vars = unlist(outer.performance[c("cum.y.inner.name", "cum.y.outer.name", "sim.y.outer.name")])
 
+  assertClass(resample.overfit, "ResampleOverfitResult", null.ok = TRUE)
+
   # what does each value mean. What is shown in the legend
   translate = c(y.inner.name = "training",
                 y.outer.name = "test",
@@ -21,15 +23,17 @@ plot.SimulateOuterPerformanceResult = function(outer.performance, resample.overf
   mdata = melt(data, measure.vars = measure.vars)
   mdata[, ':='(do.summary = get("variable") %in% summary.vars), ]
 
-  # data for boxplots
-  data.boxplots = rbind(
-    data.table(value = resample.overfit$tuning.resampled$measures.test[,2], dob = max(data$dob) + 1, variable = outer.performance$y.outer.name),
-    data.table(value = resample.overfit$untuned.resampled$measures.test[,2], dob = 0, variable = "untuned"))
-
   # renaming stuff
   mdata[, variable := plyr::revalue(variable, replace = translate, warn_missing = FALSE)]
-  data.boxplots[, variable := plyr::revalue(variable, replace = translate, warn_missing = FALSE)]
 
+  # data for boxplots
+  if (!is.null(resample.overfit)) {
+    data.boxplots = rbind(
+      data.table(value = resample.overfit$tuning.resampled$measures.test[,2], dob = max(data$dob) + 1, variable = outer.performance$y.outer.name),
+      data.table(value = resample.overfit$untuned.resampled$measures.test[,2], dob = 0, variable = "untuned"))
+    # renaming stuff
+    data.boxplots[, variable := plyr::revalue(variable, replace = translate, warn_missing = FALSE)]
+  }
 
   g = ggplot(mapping = aes_string(x = "dob", y = "value", color = "variable"))
   g = g + stat_summary(data = mdata[get("do.summary") == TRUE, ], fun.y=median, geom="line")

diff --git a/tests/testthat/test_all.R b/tests/testthat/test_all.R
@@ -3,33 +3,42 @@ context("Workflow")
 test_that("basic workflow works", {
   task = iris.task
   learner = makeLearner("classif.ksvm")
-  par.set = makeParamSet(
-    makeNumericParam(id = "C",  upper = 10, lower = -5, trafo = function(x) 2^x),
-    makeNumericParam(id = "sigma",  upper = 15, lower = -15, trafo = function(x) 2^x)
+  par.sets = list(
+    makeParamSet(
+      makeNumericParam(id = "C",  upper = 10, lower = -5, trafo = function(x) 2^x),
+      makeNumericParam(id = "sigma",  upper = 15, lower = -15, trafo = function(x) 2^x)
+    ),
+    makeParamSet(
+      makeDiscreteParam(id = "C", values = c(0.1, 10))
+    )
   )
-  tune.control = makeTuneControlRandom(maxit = 5)
-  learner.tuned = makeTuneWrapper(learner = learner, resampling = hout, par.set = par.set, control = tune.control)
-  ro = resampleOverfit(learner = learner.tuned, task = task, resampling = cv3)
 
-  expect_class(ro, "ResampleOverfitResult")
-  expect_list(ro[c("tuning.resampled", "untuned.resampled")], types = "ResampleResult")
-  expect_class(ro$task, "Task")
-  expect_list(ro$measures, "Measure")
+  for (par.set in par.sets) {
+    tune.control = makeTuneControlRandom(maxit = 5)
+    learner.tuned = makeTuneWrapper(learner = learner, resampling = hout, par.set = par.set, control = tune.control)
+    ro = resampleOverfit(learner = learner.tuned, task = task, resampling = cv3)
 
-  outer.errors = calcOuterPerformances(ro, only.on.improvement = FALSE)
-  outer.errors = calcOuterPerformances(ro, only.on.improvement = TRUE)
+    expect_class(ro, "ResampleOverfitResult")
+    expect_list(ro[c("tuning.resampled", "untuned.resampled")], types = "ResampleResult")
+    expect_class(ro$task, "Task")
+    expect_list(ro$measures, "Measure")
 
-  expect_class(outer.errors, "OuterPerformanceResult")
-  expect_class(outer.errors$data, "data.table")
-  expect_list(outer.errors$measures, "Measure")
-  expect_character(outer.errors$y.inner.name)
-  expect_character(outer.errors$y.outer.name)
-  expect_subset(c(outer.errors$y.inner.name, outer.errors$y.outer.name, "dob", "iter"), colnames(outer.errors$data))
-  expect_set_equal(unique(outer.errors$data$dob), 1:5)
-  expect_set_equal(unique(outer.errors$data$iter), 1:3)
+    outer.errors = calcOuterPerformances(ro, only.on.improvement = FALSE)
+    outer.errors = calcOuterPerformances(ro, only.on.improvement = TRUE)
 
-  outer.errors = simulateOuterPerformance(outer.errors)
-  expect_numeric(outer.errors$data[[outer.errors$sim.y.outer.name]], any.missing = FALSE)
+    expect_class(outer.errors, "OuterPerformanceResult")
+    expect_class(outer.errors$data, "data.table")
+    expect_list(outer.errors$measures, "Measure")
+    expect_character(outer.errors$y.inner.name)
+    expect_character(outer.errors$y.outer.name)
+    expect_subset(c(outer.errors$y.inner.name, outer.errors$y.outer.name, "dob", "iter"), colnames(outer.errors$data))
+    expect_set_equal(unique(outer.errors$data$dob), 1:5)
+    expect_set_equal(unique(outer.errors$data$iter), 1:3)
 
-  plot(outer.errors, ro)
+    outer.errors = simulateOuterPerformance(outer.errors)
+    expect_numeric(outer.errors$data[[outer.errors$sim.y.outer.name]], any.missing = FALSE)
+
+    plot(outer.errors, ro)
+    plot(outer.errors)
+  }
 })