add example using the with foreach::foreach() in tune_bayes

[brunocarlin]

Can you guys show how to do it in the examples of the function?

[topepo]

Can you be more specific? That function is used internally. Do you mean how to use parallel processing?

[brunocarlin]

Yes for example I use

cl <- makeForkCluster(16)
registerDoParallel(cl)
foreach::getDoParWorkers()

search_res <-
  tune_bayes(
    svm_wflow,
    resamples = folds,param_info = svm_set,
    # Generate five at semi-random to start
    initial = 5,
    iter = 30,
    # How to measure performance?
    metrics = metric_set(roc_auc),
    control = control_bayes(no_improve = 20, verbose = TRUE)
  )

And it works fine on Linux, on windows or using makeCluster the tune_bayes fails

[topepo]

Forking doesn't work outside of linux and macOS. For Windows, you could use

library(doParallel)
cl <- makePSOCKcluster(parallel::detectCores(logical = FALSE))
registerDoParallel(cl)

It is very much worth looking at the doFuture package. It has some nice features that work on top of foreach.

[brunocarlin]

Really cool package(doParallel), I will try it on windows tomorrow, if it works I will close the thread, if it doesn't I will try to make some reprex, sorry for the inconvenience it is so cool using this package (tune and tidymodels in general)

[brunocarlin]

It seems Windowns fails somehow to use multiple workers

library(tidymodels)
#> Registered S3 method overwritten by 'xts':
#>   method     from
#>   as.zoo.xts zoo
#> -- Attaching packages ---------------------------------------------------- tidymodels 0.0.3 --
#> v broom     0.5.3     v purrr     0.3.3
#> v dials     0.0.4     v recipes   0.1.9
#> v dplyr     0.8.3     v rsample   0.0.5
#> v ggplot2   3.2.1     v tibble    2.1.3
#> v infer     0.5.1     v yardstick 0.0.4
#> v parsnip   0.0.5
#> -- Conflicts ------------------------------------------------------- tidymodels_conflicts() --
#> x purrr::discard()    masks scales::discard()
#> x dplyr::filter()     masks stats::filter()
#> x dplyr::lag()        masks stats::lag()
#> x ggplot2::margin()   masks dials::margin()
#> x recipes::step()     masks stats::step()
#> x recipes::yj_trans() masks scales::yj_trans()
library(tune)
library(kernlab)
#> 
#> Attaching package: 'kernlab'
#> The following object is masked from 'package:purrr':
#> 
#>     cross
#> The following object is masked from 'package:ggplot2':
#> 
#>     alpha
#> The following object is masked from 'package:scales':
#> 
#>     alpha

# Load data
load(url("http://bit.ly/seg-data"))

segmentationData <-
  segmentationData %>%
  select(-Case, -Cell, -contains("Centroid"))

set.seed(8567)
tr_te_split <- initial_split(segmentationData)

seg_train <- training(tr_te_split)
seg_test  <-  testing(tr_te_split)

set.seed(5121)
folds <- vfold_cv(seg_train, repeats = 2)


seg_pre_proc <-
  recipe(Class ~ ., data = seg_train) %>%
  step_YeoJohnson(all_predictors()) %>%
  step_normalize(all_predictors()) %>%
  step_pca(all_predictors(), num_comp = tune()) %>%
  step_downsample(Class)

svm_mod <-
  svm_rbf(mode = "classification", cost = tune(), rbf_sigma = tune()) %>%
  set_engine("kernlab")


library(workflows) 

svm_wflow <-
  workflow() %>%
  add_model(svm_mod) %>%
  add_recipe(seg_pre_proc)


svm_set <- parameters(svm_wflow)


svm_set <- svm_set %>% update(num_comp = num_comp(c(1, 20)))


set.seed(1291)

library(doParallel)
#> Loading required package: foreach
#> 
#> Attaching package: 'foreach'
#> The following objects are masked from 'package:purrr':
#> 
#>     accumulate, when
#> Loading required package: iterators
#> Loading required package: parallel
cl <- makePSOCKcluster(parallel::detectCores(logical = FALSE))
registerDoParallel(cl)
foreach::getDoParWorkers()
#> [1] 2

search_res <- 
  tune_bayes(
    svm_wflow, 
    resamples = folds,
    # To use non-default parameter ranges
    param_info = svm_set,
    # Generate five at semi-random to start
    initial = 3,
    iter = 30,
    # How to measure performance?
    metrics = metric_set(roc_auc),
    control = control_bayes(no_improve = 20, verbose = TRUE)
  )
#> 
#> >  Generating a set of 3 initial parameter results
#> Warning: All models failed in tune_grid(). See the `.notes` column.
#> v Initialization complete
#> 
#> Error: All of the models failed.

^{Created on 2020-01-25 by the reprex package (v0.3.0)}

Session info

devtools::session_info()
#> - Session info ---------------------------------------------------------------
#>  setting  value                       
#>  version  R version 3.6.2 (2019-12-12)
#>  os       Windows 10 x64              
#>  system   x86_64, mingw32             
#>  ui       RTerm                       
#>  language (EN)                        
#>  collate  English_United States.1252  
#>  ctype    English_United States.1252  
#>  tz       America/Sao_Paulo           
#>  date     2020-01-25                  
#> 
#> - Packages -------------------------------------------------------------------
#>  package       * version    date       lib source                          
#>  assertthat      0.2.1      2019-03-21 [2] CRAN (R 3.6.2)                  
#>  backports       1.1.5      2019-10-02 [2] CRAN (R 3.6.1)                  
#>  base64enc       0.1-3      2015-07-28 [2] CRAN (R 3.6.0)                  
#>  bayesplot       1.7.1      2019-12-01 [2] CRAN (R 3.6.2)                  
#>  boot            1.3-24     2019-12-20 [2] CRAN (R 3.6.2)                  
#>  broom         * 0.5.3      2019-12-14 [2] CRAN (R 3.6.2)                  
#>  callr           3.4.0      2019-12-09 [2] CRAN (R 3.6.2)                  
#>  class           7.3-15     2019-01-01 [2] CRAN (R 3.6.2)                  
#>  cli             2.0.1      2020-01-08 [2] CRAN (R 3.6.2)                  
#>  codetools       0.2-16     2018-12-24 [2] CRAN (R 3.6.0)                  
#>  colorspace      1.4-1      2019-03-18 [2] CRAN (R 3.6.1)                  
#>  colourpicker    1.0        2017-09-27 [2] CRAN (R 3.6.2)                  
#>  crayon          1.3.4      2017-09-16 [2] CRAN (R 3.6.2)                  
#>  crosstalk       1.0.0      2016-12-21 [2] CRAN (R 3.6.2)                  
#>  desc            1.2.0      2018-05-01 [2] CRAN (R 3.6.2)                  
#>  devtools        2.2.1      2019-09-24 [2] CRAN (R 3.6.2)                  
#>  dials         * 0.0.4      2019-12-02 [2] CRAN (R 3.6.2)                  
#>  DiceDesign      1.8-1      2019-07-31 [2] CRAN (R 3.6.2)                  
#>  digest          0.6.23     2019-11-23 [2] CRAN (R 3.6.2)                  
#>  doParallel    * 1.0.15     2019-08-02 [1] CRAN (R 3.6.2)                  
#>  dplyr         * 0.8.3      2019-07-04 [2] CRAN (R 3.6.2)                  
#>  DT              0.11       2019-12-19 [2] CRAN (R 3.6.2)                  
#>  dygraphs        1.1.1.6    2018-07-11 [2] CRAN (R 3.6.2)                  
#>  ellipsis        0.3.0      2019-09-20 [2] CRAN (R 3.6.2)                  
#>  evaluate        0.14       2019-05-28 [2] CRAN (R 3.6.2)                  
#>  fansi           0.4.1      2020-01-08 [2] CRAN (R 3.6.2)                  
#>  fastmap         1.0.1      2019-10-08 [2] CRAN (R 3.6.2)                  
#>  foreach       * 1.4.7      2019-07-27 [2] CRAN (R 3.6.2)                  
#>  fs              1.3.1      2019-05-06 [2] CRAN (R 3.6.2)                  
#>  furrr           0.1.0      2018-05-16 [2] CRAN (R 3.6.2)                  
#>  future          1.15.1     2019-11-25 [2] CRAN (R 3.6.2)                  
#>  generics        0.0.2      2018-11-29 [2] CRAN (R 3.6.2)                  
#>  ggplot2       * 3.2.1      2019-08-10 [2] CRAN (R 3.6.2)                  
#>  ggridges        0.5.2      2020-01-12 [2] CRAN (R 3.6.2)                  
#>  globals         0.12.5     2019-12-07 [2] CRAN (R 3.6.1)                  
#>  glue            1.3.1      2019-03-12 [2] CRAN (R 3.6.2)                  
#>  gower           0.2.1      2019-05-14 [2] CRAN (R 3.6.1)                  
#>  GPfit           1.0-8      2019-02-08 [2] CRAN (R 3.6.2)                  
#>  gridExtra       2.3        2017-09-09 [2] CRAN (R 3.6.2)                  
#>  gtable          0.3.0      2019-03-25 [2] CRAN (R 3.6.2)                  
#>  gtools          3.8.1      2018-06-26 [2] CRAN (R 3.6.0)                  
#>  hardhat         0.1.1      2020-01-08 [2] CRAN (R 3.6.2)                  
#>  highr           0.8        2019-03-20 [2] CRAN (R 3.6.2)                  
#>  htmltools       0.4.0      2019-10-04 [2] CRAN (R 3.6.2)                  
#>  htmlwidgets     1.5.1      2019-10-08 [2] CRAN (R 3.6.2)                  
#>  httpuv          1.5.2      2019-09-11 [2] CRAN (R 3.6.2)                  
#>  igraph          1.2.4.2    2019-11-27 [2] CRAN (R 3.6.2)                  
#>  infer         * 0.5.1      2019-11-19 [2] CRAN (R 3.6.2)                  
#>  inline          0.3.15     2018-05-18 [2] CRAN (R 3.6.2)                  
#>  ipred           0.9-9      2019-04-28 [2] CRAN (R 3.6.2)                  
#>  iterators     * 1.0.12     2019-07-26 [2] CRAN (R 3.6.2)                  
#>  janeaustenr     0.1.5      2017-06-10 [2] CRAN (R 3.6.2)                  
#>  kernlab       * 0.9-29     2019-11-12 [1] CRAN (R 3.6.1)                  
#>  knitr           1.26       2019-11-12 [2] CRAN (R 3.6.2)                  
#>  later           1.0.0      2019-10-04 [2] CRAN (R 3.6.2)                  
#>  lattice         0.20-38    2018-11-04 [2] CRAN (R 3.6.2)                  
#>  lava            1.6.6      2019-08-01 [2] CRAN (R 3.6.2)                  
#>  lazyeval        0.2.2      2019-03-15 [2] CRAN (R 3.6.2)                  
#>  lhs             1.0.1      2019-02-03 [2] CRAN (R 3.6.2)                  
#>  lifecycle       0.1.0      2019-08-01 [2] CRAN (R 3.6.2)                  
#>  listenv         0.8.0      2019-12-05 [2] CRAN (R 3.6.2)                  
#>  lme4            1.1-21     2019-03-05 [2] CRAN (R 3.6.2)                  
#>  loo             2.2.0      2019-12-19 [2] CRAN (R 3.6.2)                  
#>  lubridate       1.7.4      2018-04-11 [2] CRAN (R 3.6.2)                  
#>  magrittr        1.5        2014-11-22 [2] CRAN (R 3.6.2)                  
#>  markdown        1.1        2019-08-07 [2] CRAN (R 3.6.2)                  
#>  MASS            7.3-51.5   2019-12-20 [2] CRAN (R 3.6.2)                  
#>  Matrix          1.2-18     2019-11-27 [2] CRAN (R 3.6.2)                  
#>  matrixStats     0.55.0     2019-09-07 [2] CRAN (R 3.6.2)                  
#>  memoise         1.1.0      2017-04-21 [2] CRAN (R 3.6.2)                  
#>  mime            0.8        2019-12-19 [2] CRAN (R 3.6.2)                  
#>  miniUI          0.1.1.1    2018-05-18 [2] CRAN (R 3.6.2)                  
#>  minqa           1.2.4      2014-10-09 [2] CRAN (R 3.6.2)                  
#>  munsell         0.5.0      2018-06-12 [2] CRAN (R 3.6.2)                  
#>  nlme            3.1-143    2019-12-10 [2] CRAN (R 3.6.2)                  
#>  nloptr          1.2.1      2018-10-03 [2] CRAN (R 3.6.2)                  
#>  nnet            7.3-12     2016-02-02 [2] CRAN (R 3.6.2)                  
#>  parsnip       * 0.0.5      2020-01-07 [2] CRAN (R 3.6.2)                  
#>  pillar          1.4.3      2019-12-20 [2] CRAN (R 3.6.2)                  
#>  pkgbuild        1.0.6      2019-10-09 [2] CRAN (R 3.6.2)                  
#>  pkgconfig       2.0.3      2019-09-22 [2] CRAN (R 3.6.2)                  
#>  pkgload         1.0.2      2018-10-29 [2] CRAN (R 3.6.2)                  
#>  plyr            1.8.5      2019-12-10 [2] CRAN (R 3.6.2)                  
#>  prettyunits     1.1.0      2020-01-09 [2] CRAN (R 3.6.2)                  
#>  pROC            1.16.1     2020-01-14 [2] CRAN (R 3.6.2)                  
#>  processx        3.4.1      2019-07-18 [2] CRAN (R 3.6.2)                  
#>  prodlim         2019.11.13 2019-11-17 [2] CRAN (R 3.6.2)                  
#>  promises        1.1.0      2019-10-04 [2] CRAN (R 3.6.2)                  
#>  ps              1.3.0      2018-12-21 [2] CRAN (R 3.6.2)                  
#>  purrr         * 0.3.3      2019-10-18 [2] CRAN (R 3.6.2)                  
#>  R6              2.4.1      2019-11-12 [2] CRAN (R 3.6.2)                  
#>  Rcpp            1.0.3      2019-11-08 [2] CRAN (R 3.6.2)                  
#>  recipes       * 0.1.9      2020-01-07 [2] CRAN (R 3.6.2)                  
#>  remotes         2.1.0      2019-06-24 [2] CRAN (R 3.6.2)                  
#>  reshape2        1.4.3      2017-12-11 [2] CRAN (R 3.6.2)                  
#>  rlang           0.4.2      2019-11-23 [2] CRAN (R 3.6.2)                  
#>  rmarkdown       2.0        2019-12-12 [2] CRAN (R 3.6.2)                  
#>  rpart           4.1-15     2019-04-12 [2] CRAN (R 3.6.2)                  
#>  rprojroot       1.3-2      2018-01-03 [2] CRAN (R 3.6.2)                  
#>  rsample       * 0.0.5      2019-07-12 [2] CRAN (R 3.6.2)                  
#>  rsconnect       0.8.16     2019-12-13 [2] CRAN (R 3.6.2)                  
#>  rstan           2.19.2     2019-07-09 [2] CRAN (R 3.6.2)                  
#>  rstanarm        2.19.2     2019-10-03 [2] CRAN (R 3.6.2)                  
#>  rstantools      2.0.0      2019-09-15 [2] CRAN (R 3.6.2)                  
#>  rstudioapi      0.10       2019-03-19 [2] CRAN (R 3.6.2)                  
#>  scales        * 1.1.0      2019-11-18 [2] CRAN (R 3.6.2)                  
#>  sessioninfo     1.1.1      2018-11-05 [2] CRAN (R 3.6.2)                  
#>  shiny           1.4.0      2019-10-10 [2] CRAN (R 3.6.2)                  
#>  shinyjs         1.1        2020-01-13 [2] CRAN (R 3.6.2)                  
#>  shinystan       2.5.0      2018-05-01 [2] CRAN (R 3.6.2)                  
#>  shinythemes     1.1.2      2018-11-06 [2] CRAN (R 3.6.2)                  
#>  SnowballC       0.6.0      2019-01-15 [2] CRAN (R 3.6.0)                  
#>  StanHeaders     2.19.0     2019-09-07 [2] CRAN (R 3.6.2)                  
#>  stringi         1.4.5      2020-01-11 [2] CRAN (R 3.6.2)                  
#>  stringr         1.4.0      2019-02-10 [2] CRAN (R 3.6.2)                  
#>  survival        3.1-8      2019-12-03 [2] CRAN (R 3.6.2)                  
#>  testthat        2.3.1      2019-12-01 [2] CRAN (R 3.6.2)                  
#>  threejs         0.3.1      2017-08-13 [2] CRAN (R 3.6.2)                  
#>  tibble        * 2.1.3      2019-06-06 [2] CRAN (R 3.6.2)                  
#>  tidymodels    * 0.0.3      2019-10-04 [2] CRAN (R 3.6.2)                  
#>  tidyposterior   0.0.2      2018-11-15 [2] CRAN (R 3.6.2)                  
#>  tidypredict     0.4.3      2019-09-03 [2] CRAN (R 3.6.2)                  
#>  tidyr         * 1.0.0      2019-09-11 [2] CRAN (R 3.6.2)                  
#>  tidyselect      0.2.5      2018-10-11 [2] CRAN (R 3.6.2)                  
#>  tidytext        0.2.2      2019-07-29 [2] CRAN (R 3.6.2)                  
#>  timeDate        3043.102   2018-02-21 [2] CRAN (R 3.6.0)                  
#>  tokenizers      0.2.1      2018-03-29 [2] CRAN (R 3.6.2)                  
#>  tune          * 0.0.1      2020-01-23 [2] Github (tidymodels/tune@ba56ec5)
#>  usethis         1.5.1      2019-07-04 [2] CRAN (R 3.6.2)                  
#>  vctrs           0.2.1      2019-12-17 [2] CRAN (R 3.6.2)                  
#>  withr           2.1.2      2018-03-15 [2] CRAN (R 3.6.2)                  
#>  workflows     * 0.1.0      2019-12-30 [2] CRAN (R 3.6.2)                  
#>  xfun            0.12       2020-01-13 [2] CRAN (R 3.6.2)                  
#>  xtable          1.8-4      2019-04-21 [2] CRAN (R 3.6.2)                  
#>  xts             0.11-2     2018-11-05 [2] CRAN (R 3.6.2)                  
#>  yaml            2.2.0      2018-07-25 [2] CRAN (R 3.6.0)                  
#>  yardstick     * 0.0.4      2019-08-26 [2] CRAN (R 3.6.2)                  
#>  zeallot         0.1.0      2018-01-28 [2] CRAN (R 3.6.2)                  
#>  zoo             1.8-7      2020-01-10 [2] CRAN (R 3.6.2)                  
#> 
#> [1] C:/Users/bcarl2/OneDrive - Monsanto/Migrated from My PC/Documents/R/win-library/3.6
#> [2] C:/Users/bcarl2/OneDrive - Monsanto/Migrated from My PC/Documents/R/R-3.6.2/library

[maxheld83]

I'm guessing the problem is that the examples in the workshop, similar to the one above, where based on the process forking used by {doMC}, and that we might have to be more explicit about how to expose pkgs and objects to the runners for parallel backends which don't do process forking.

({doMC}, only available for unix-compatible systems such as macOS and Linux uses process forking, which (more or less) makes the entire environment (vars, pkgs) available to all the parallel processes.)

I don't know enough about the internals of {tune} or parallel processing on windows to know for sure whether we can just fix this with a bunch of !! or whatever.

[brunocarlin]

Solved it using:

clusterEvalQ(cl, {library(tidymodels)})

library(tidymodels)
#> -- Attaching packages ----------------------------------------------------------------------------- tidymodels 0.0.3 --
#> v broom     0.5.4     v purrr     0.3.3
#> v dials     0.0.4     v recipes   0.1.9
#> v dplyr     0.8.3     v rsample   0.0.5
#> v ggplot2   3.2.1     v tibble    2.1.3
#> v infer     0.5.1     v yardstick 0.0.5
#> v parsnip   0.0.5
#> -- Conflicts -------------------------------------------------------------------------------- tidymodels_conflicts() --
#> x purrr::discard()    masks scales::discard()
#> x dplyr::filter()     masks stats::filter()
#> x dplyr::lag()        masks stats::lag()
#> x ggplot2::margin()   masks dials::margin()
#> x recipes::step()     masks stats::step()
#> x recipes::yj_trans() masks scales::yj_trans()
library(tune)
library(kernlab)
#> 
#> Attaching package: 'kernlab'
#> The following object is masked from 'package:purrr':
#> 
#>     cross
#> The following object is masked from 'package:ggplot2':
#> 
#>     alpha
#> The following object is masked from 'package:scales':
#> 
#>     alpha

load(url("http://bit.ly/seg-data"))

segmentationData <-
  segmentationData %>%
  select(-Case, -Cell, -contains("Centroid"))

set.seed(8567)
tr_te_split <- initial_split(segmentationData)

seg_train <- training(tr_te_split)
seg_test  <-  testing(tr_te_split)

set.seed(5121)
folds <- vfold_cv(seg_train, repeats = 2)


seg_pre_proc <-
  recipe(Class ~ ., data = seg_train) %>%
  step_YeoJohnson(all_predictors()) %>%
  step_normalize(all_predictors()) %>%
  step_pca(all_predictors(), num_comp = tune()) %>%
  step_downsample(Class)

svm_mod <-
  svm_rbf(mode = "classification", cost = tune(), rbf_sigma = tune()) %>%
  set_engine("kernlab")


library(workflows) 

svm_wflow <-
  workflow() %>%
  add_model(svm_mod) %>%
  add_recipe(seg_pre_proc)


svm_set <- parameters(svm_wflow)


svm_set <- svm_set %>% update(num_comp = num_comp(c(1, 20)))


set.seed(1291)

library(doParallel)
#> Loading required package: foreach
#> 
#> Attaching package: 'foreach'
#> The following objects are masked from 'package:purrr':
#> 
#>     accumulate, when
#> Loading required package: iterators
#> Loading required package: parallel
cl <- makePSOCKcluster(parallel::detectCores(logical = FALSE))
registerDoParallel(cl)

foreach::getDoParWorkers()
#> [1] 2

clusterEvalQ(cl, {library(tidymodels)})
#> [[1]]
#>  [1] "yardstick"  "tibble"     "rsample"    "tidyr"      "recipes"   
#>  [6] "purrr"      "parsnip"    "infer"      "ggplot2"    "dplyr"     
#> [11] "dials"      "scales"     "broom"      "tidymodels" "stats"     
#> [16] "graphics"   "grDevices"  "utils"      "datasets"   "methods"   
#> [21] "base"      
#> 
#> [[2]]
#>  [1] "yardstick"  "tibble"     "rsample"    "tidyr"      "recipes"   
#>  [6] "purrr"      "parsnip"    "infer"      "ggplot2"    "dplyr"     
#> [11] "dials"      "scales"     "broom"      "tidymodels" "stats"     
#> [16] "graphics"   "grDevices"  "utils"      "datasets"   "methods"   
#> [21] "base"


tune_bayes(
    svm_wflow, 
    resamples = folds,
    # To use non-default parameter ranges
    param_info = svm_set,
    # Generate five at semi-random to start
    initial = 3,
    iter = 5,
    # How to measure performance?
    metrics = metric_set(roc_auc),
    control = control_bayes(no_improve = 20, verbose = TRUE)
  )
#> 
#> >  Generating a set of 3 initial parameter results
#> v Initialization complete
#> 
#> Optimizing roc_auc using the expected improvement
#> 
#> -- Iteration 1 --------------------------------------------------------------------------------------------------------
#> 
#> i Current best:      roc_auc=0.8653 (@iter 0)
#> i Gaussian process model
#> v Gaussian process model
#> i Generating 5000 candidates
#> i Predicted candidates
#> i cost=0.00382, rbf_sigma=5.23e-06, num_comp=3
#> i Estimating performance
#> v Estimating performance
#> (x) Newest results:  roc_auc=0.8521 (+/-0.00759)
#> 
#> -- Iteration 2 --------------------------------------------------------------------------------------------------------
#> 
#> i Current best:      roc_auc=0.8653 (@iter 0)
#> i Gaussian process model
#> v Gaussian process model
#> i Generating 5000 candidates
#> i Predicted candidates
#> i cost=30.3, rbf_sigma=1.2e-10, num_comp=20
#> i Estimating performance
#> v Estimating performance
#> (x) Newest results:  roc_auc=0.865 (+/-0.00694)
#> 
#> -- Iteration 3 --------------------------------------------------------------------------------------------------------
#> 
#> i Current best:      roc_auc=0.8653 (@iter 0)
#> i Gaussian process model
#> v Gaussian process model
#> i Generating 5000 candidates
#> i Predicted candidates
#> i cost=0.021, rbf_sigma=0.996, num_comp=3
#> i Estimating performance
#> v Estimating performance
#> (x) Newest results:  roc_auc=0.8562 (+/-0.00732)
#> 
#> -- Iteration 4 --------------------------------------------------------------------------------------------------------
#> 
#> i Current best:      roc_auc=0.8653 (@iter 0)
#> i Gaussian process model
#> v Gaussian process model
#> i Generating 5000 candidates
#> i Predicted candidates
#> i cost=0.00168, rbf_sigma=1.67e-10, num_comp=20
#> i Estimating performance
#> v Estimating performance
#> (x) Newest results:  roc_auc=0.7917 (+/-0.034)
#> 
#> -- Iteration 5 --------------------------------------------------------------------------------------------------------
#> 
#> i Current best:      roc_auc=0.8653 (@iter 0)
#> i Gaussian process model
#> v Gaussian process model
#> i Generating 5000 candidates
#> i Predicted candidates
#> i cost=0.001, rbf_sigma=1.27e-09, num_comp=8
#> i Estimating performance
#> v Estimating performance
#> (x) Newest results:  roc_auc=0.7922 (+/-0.0341)
#> #  10-fold cross-validation repeated 2 times 
#> # A tibble: 120 x 6
#>    splits             id      id2    .metrics         .notes           .iter
#>  * <list>             <chr>   <chr>  <list>           <list>           <dbl>
#>  1 <split [1.4K/152]> Repeat1 Fold01 <tibble [3 x 6]> <tibble [0 x 1]>     0
#>  2 <split [1.4K/152]> Repeat1 Fold02 <tibble [3 x 6]> <tibble [0 x 1]>     0
#>  3 <split [1.4K/152]> Repeat1 Fold03 <tibble [3 x 6]> <tibble [0 x 1]>     0
#>  4 <split [1.4K/152]> Repeat1 Fold04 <tibble [3 x 6]> <tibble [0 x 1]>     0
#>  5 <split [1.4K/152]> Repeat1 Fold05 <tibble [3 x 6]> <tibble [0 x 1]>     0
#>  6 <split [1.4K/151]> Repeat1 Fold06 <tibble [3 x 6]> <tibble [0 x 1]>     0
#>  7 <split [1.4K/151]> Repeat1 Fold07 <tibble [3 x 6]> <tibble [0 x 1]>     0
#>  8 <split [1.4K/151]> Repeat1 Fold08 <tibble [3 x 6]> <tibble [0 x 1]>     0
#>  9 <split [1.4K/151]> Repeat1 Fold09 <tibble [3 x 6]> <tibble [0 x 1]>     0
#> 10 <split [1.4K/151]> Repeat1 Fold10 <tibble [3 x 6]> <tibble [0 x 1]>     0
#> # ... with 110 more rows

^{Created on 2020-02-03 by the reprex package (v0.3.0)}