diff --git a/DESCRIPTION b/DESCRIPTION index 1db4664d6..d3ea4dc1e 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: tune Title: Tidy Tuning Tools -Version: 1.1.2.9001 +Version: 1.1.2.9002 Authors@R: c( person("Max", "Kuhn", , "max@posit.co", role = c("aut", "cre"), comment = c(ORCID = "0000-0003-2402-136X")), diff --git a/NAMESPACE b/NAMESPACE index af15732e9..099354149 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -136,6 +136,7 @@ export(.catch_and_log) export(.catch_and_log_fit) export(.config_key_from_metrics) export(.estimate_metrics) +export(.filter_perf_metrics) export(.get_extra_col_names) export(.get_fingerprint) export(.get_tune_eval_times) @@ -156,6 +157,7 @@ export(check_parameters) export(check_rset) export(check_time) export(check_workflow) +export(choose_eval_time) export(choose_metric) export(collect_extracts) export(collect_metrics) diff --git a/R/metric-selection.R b/R/metric-selection.R index c3f5bce4c..6fc1e7064 100644 --- a/R/metric-selection.R +++ b/R/metric-selection.R @@ -4,6 +4,8 @@ #' @param metric A character value for which metric is being used. #' @param eval_time An optional vector of times to compute dynamic and/or #' integrated metrics. +#' @param x An object with class `tune_results`. +#' @param call The call to be displayed in warnings or errors. #' @description #' These are developer-facing functions used to compute and validate choices #' for performance metrics. For survival analysis models, there are similar @@ -15,6 +17,14 @@ #' no value is given by the user, the first metric value is used (with a #' warning). #' +#' For evaluation times, one is only required when the metric type is dynamic +#' (e.g. [yardstick::brier_survival()] or [yardstick::roc_auc_survival()]). For +#' these metrics, we require a single numeric value that was originally given +#' to the function used to produce `x` (such as [tune_grid()]). +#' +#' If a time is required and none is given, the first value in the vector +#' originally given in the `eval_time` argument is used (with a warning). +#' #' @keywords internal #' @export choose_metric <- function(x, metric, ..., call = rlang::caller_env()) { @@ -25,10 +35,12 @@ choose_metric <- function(x, metric, ..., call = rlang::caller_env()) { if (is.null(metric)) { metric <- mtr_info$metric[1] - cli::cli_warn("No value of {.arg metric} was given; {.val {metric}} will be used.", call = call) + cli::cli_warn("No value of {.arg metric} was given; {.val {metric}} + will be used.", + call = call) } else { metric <- check_mult_metrics(metric, call = call) - check_right_metric(mtr_info, metric, call = call) + check_metric_in_tune_results(mtr_info, metric, call = call) } mtr_info[mtr_info$metric == metric,] @@ -40,16 +52,19 @@ check_mult_metrics <- function(metric, ..., call = rlang::caller_env()) { num_metrics <- length(metric) metric <- metric[1] if (num_metrics > 1) { - cli::cli_warn("{num_metrics} metric{?s} were given; {.val {metric}} will be used.", call = call) + cli::cli_warn("{num_metrics} metric{?s} were given; {.val {metric}} will + be used.", + call = call) } metric } -check_right_metric <- function(mtr_info, metric, ..., call = rlang::caller_env()) { +check_metric_in_tune_results <- function(mtr_info, metric, ..., call = rlang::caller_env()) { rlang::check_dots_empty() if (!any(mtr_info$metric == metric)) { - cli::cli_abort("{.val {metric}} was not in the metric set. Please choose from: {.val {mtr_info$metric}}.", call = call) + cli::cli_abort("{.val {metric}} was not in the metric set. Please choose + from: {.val {mtr_info$metric}}.", call = call) } invisible(NULL) } @@ -58,6 +73,53 @@ contains_survival_metric <- function(mtr_info) { any(grepl("_survival", mtr_info$class)) } +#' @rdname choose_metric +#' @export +choose_eval_time <- function(x, metric, eval_time = NULL, ..., call = rlang::caller_env()) { + rlang::check_dots_empty() + + mtr_set <- .get_tune_metrics(x) + mtr_info <- tibble::as_tibble(mtr_set) + + if (!contains_survival_metric(mtr_info)) { + if (!is.null(eval_time)) { + cli::cli_warn("Evaluation times are only required when the model + mode is {.val censored regression} (and will be ignored).") + } + return(NULL) + } + + # If we need an eval time, set it to the possible values so that + # we can choose the first value + if (is_dyn(mtr_set, metric) && is.null(eval_time)) { + eval_time <- .get_tune_eval_times(x) + } + + eval_time <- first_eval_time(mtr_set, metric = metric, eval_time = eval_time) + + check_eval_time_in_tune_results(x, eval_time, call = call) + + eval_time +} + +is_dyn <- function(mtr_set, metric) { + mtr_info <- tibble::as_tibble(mtr_set) + mtr_cls <- mtr_info$class[mtr_info$metric == metric] + mtr_cls == "dynamic_survival_metric" +} + +check_eval_time_in_tune_results <- function(x, eval_time, call = rlang::caller_env()) { + given_times <- .get_tune_eval_times(x) + if (!is.null(eval_time)) { + if (!any(eval_time == given_times)) { + print_time <- format(eval_time, digits = 3) + cli::cli_abort("Evaluation time {print_time} is not in the results.", + call = call) + } + } + invisible(NULL) +} + # ------------------------------------------------------------------------------ #' @rdname choose_metric @@ -88,7 +150,9 @@ first_eval_time <- function(mtr_set, metric = NULL, eval_time = NULL) { no_time_req <- c("static_survival_metric", "integrated_survival_metric") if (mtr_info$class %in% no_time_req) { if (num_times > 0) { - cli::cli_warn("Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric.") + cli::cli_warn("Evaluation times are only required when dynmanic or + integrated metrics are selected as the primary metric + (and will be ignored).") } return(NULL) } @@ -99,8 +163,41 @@ first_eval_time <- function(mtr_set, metric = NULL, eval_time = NULL) { } else if ( num_times > 1 ) { eval_time <- eval_time[1] print_time <- format(eval_time, digits = 3) - cli::cli_warn("{num_times} evaluation times were available; the first ({print_time}) will be used.") + cli::cli_warn("{.val {num_times}} evaluation times were specified during + tuning; the first ({print_time}) will be used.") } eval_time } + +# ------------------------------------------------------------------------------ + +#' @rdname choose_metric +#' @export +.filter_perf_metrics <- function(x, metric, eval_time) { + summary_res <- estimate_tune_results(x) + summary_res <- summary_res[summary_res$.metric == metric, ] + is_missing_mean <- is.na(summary_res$mean) + summary_res <- summary_res[!is_missing_mean, ] + + if (!is.null(eval_time) && any(colnames(summary_res) == ".eval_time")) { + summary_res <- summary_res[summary_res$.eval_time == eval_time, ] + } + if (nrow(summary_res) == 0) { + cli::cli_abort("No results are available. Please use {.fun collect_metrics} + to see if there were any issues.") + } + + summary_res +} + +# TODO will be removed shortly + +middle_eval_time <- function(x) { + x <- x[!is.na(x)] + times <- unique(x) + med_time <- median(x, na.rm = TRUE) + ind <- which.min(abs(times - med_time)) + eval_time <- times[ind] + eval_time +} diff --git a/R/plots.R b/R/plots.R index 705b9469d..c91287154 100644 --- a/R/plots.R +++ b/R/plots.R @@ -158,6 +158,7 @@ get_param_label <- function(x, id_val) { res } +# TODO remove this. default_eval_time <- function(eval_time, x, call = rlang::caller_env()) { if (!any(names(x) == ".eval_time")) { if (!is.null(eval_time)) { diff --git a/R/select_best.R b/R/select_best.R index 651846da7..47a37d16c 100644 --- a/R/select_best.R +++ b/R/select_best.R @@ -79,23 +79,9 @@ show_best.tune_results <- function(x, metric = NULL, n = 5, eval_time = NULL, .. metric_info <- choose_metric(x, metric) metric <- metric_info$metric - summary_res <- estimate_tune_results(x) + eval_time <- choose_eval_time(x, metric, eval_time = eval_time) - # TODO - metrics <- unique(summary_res$.metric) - if (length(metrics) == 1) { - metric <- metrics - } - - # get estimates/summarise - summary_res <- summary_res %>% dplyr::filter(.metric == metric) - - # TODO split selecting the req time and seeing if it is in the data - summary_res <- choose_eval_time(summary_res, x, eval_time) - - if (nrow(summary_res) == 0) { - rlang::abort("No results are available. Please check the value of `metric`.") - } + summary_res <- .filter_perf_metrics(x, metric, eval_time) if (metric_info$direction == "maximize") { summary_res <- summary_res %>% dplyr::arrange(dplyr::desc(mean)) @@ -155,30 +141,23 @@ select_by_pct_loss.tune_results <- function(x, ..., metric = NULL, limit = 2, ev param_names <- .get_tune_parameter_names(x) - dots <- rlang::enquos(...) - if (length(dots) == 0) { - rlang::abort("Please choose at least one tuning parameter to sort in `...`.") - } + check_select_dots(...) - res <- - collect_metrics(x) %>% - dplyr::filter(.metric == !!metric & !is.na(mean)) - res <- choose_eval_time(res, x, eval_time) + eval_time <- choose_eval_time(x, metric, eval_time = eval_time) + + summary_res <- .filter_perf_metrics(x, metric, eval_time) - if (nrow(res) == 0) { - rlang::abort("No results are available. Please check the value of `metric`.") - } if (metric_info$direction == "maximize") { - best_metric <- max(res$mean, na.rm = TRUE) + best_metric <- max(summary_res$mean, na.rm = TRUE) } else if (metric_info$direction == "minimize") { - best_metric <- min(res$mean, na.rm = TRUE) + best_metric <- min(summary_res$mean, na.rm = TRUE) } else if (metric_info$direction == "zero") { - which_min <- which.min(abs(res$mean)) - best_metric <- res$mean[which_min] + which_min <- which.min(abs(summary_res$mean)) + best_metric <- summary_res$mean[which_min] } - res <- - res %>% + summary_res <- + summary_res %>% dplyr::rowwise() %>% dplyr::mutate( .best = best_metric, @@ -188,8 +167,10 @@ select_by_pct_loss.tune_results <- function(x, ..., metric = NULL, limit = 2, ev ) %>% dplyr::ungroup() - res <- try(dplyr::arrange(res, !!!dots), silent = TRUE) - if (inherits(res, "try-error")) { + + dots <- rlang::enquos(...) + summary_res <- try(dplyr::arrange(summary_res, !!!dots), silent = TRUE) + if (inherits(summary_res, "try-error")) { var_nm <- rlang::eval_tidy(dots) var_nm <- purrr::map_chr(var_nm, ~ rlang::quo_name(.x)) var_nm <- var_nm[!var_nm %in% colnames(collect_metrics(x))] @@ -198,8 +179,8 @@ select_by_pct_loss.tune_results <- function(x, ..., metric = NULL, limit = 2, ev # discard models more complex than the best and # remove models with greater increase in loss than the limit - best_index <- which(res$.loss == 0) - res %>% + best_index <- which(summary_res$.loss == 0) + summary_res %>% dplyr::slice(1:best_index) %>% dplyr::filter(.loss < limit) %>% dplyr::slice(1) %>% @@ -226,49 +207,41 @@ select_by_one_std_err.tune_results <- function(x, ..., metric = NULL, eval_time param_names <- .get_tune_parameter_names(x) - dots <- rlang::enquos(...) - if (length(dots) == 0) { - rlang::abort("Please choose at least one tuning parameter to sort in `...`.") - } + check_select_dots(...) - res <- - collect_metrics(x) %>% - dplyr::filter(.metric == !!metric & !is.na(mean)) - res <- choose_eval_time(res, x, eval_time) + eval_time <- choose_eval_time(x, metric, eval_time = eval_time) - if (nrow(res) == 0) { - rlang::abort("No results are available. Please check the value of `metric`.") - } + summary_res <- .filter_perf_metrics(x, metric, eval_time) if (metric_info$direction == "maximize") { - best_index <- which.max(res$mean) - best <- res$mean[best_index] - bound <- best - res$std_err[best_index] - res <- - res %>% + best_index <- which.max(summary_res$mean) + best <- summary_res$mean[best_index] + bound <- best - summary_res$std_err[best_index] + summary_res <- + summary_res %>% dplyr::mutate( .best = best, .bound = bound ) %>% dplyr::filter(mean >= .bound) } else if (metric_info$direction == "minimize") { - best_index <- which.min(res$mean) - best <- res$mean[best_index] - bound <- best + res$std_err[best_index] - res <- - res %>% + best_index <- which.min(summary_res$mean) + best <- summary_res$mean[best_index] + bound <- best + summary_res$std_err[best_index] + summary_res <- + summary_res %>% dplyr::mutate( .best = best, .bound = bound ) %>% dplyr::filter(mean <= .bound) } else if (metric_info$direction == "zero") { - best_index <- which.min(abs(res$mean)) - best <- res$mean[best_index] - bound_lower <- -abs(best) - res$std_err[best_index] - bound_upper <- abs(best) + res$std_err[best_index] - res <- - res %>% + best_index <- which.min(abs(summary_res$mean)) + best <- summary_res$mean[best_index] + bound_lower <- -abs(best) - summary_res$std_err[best_index] + bound_upper <- abs(best) + summary_res$std_err[best_index] + summary_res <- + summary_res %>% dplyr::rowwise() %>% dplyr::mutate( .best = best, @@ -278,59 +251,24 @@ select_by_one_std_err.tune_results <- function(x, ..., metric = NULL, eval_time dplyr::ungroup() } - res <- try(dplyr::arrange(res, !!!dots), silent = TRUE) - if (inherits(res, "try-error")) { + dots <- rlang::enquos(...) + summary_res <- try(dplyr::arrange(summary_res, !!!dots), silent = TRUE) + if (inherits(summary_res, "try-error")) { var_nm <- rlang::eval_tidy(dots) var_nm <- purrr::map_chr(var_nm, ~ rlang::quo_name(.x)) var_nm <- var_nm[!var_nm %in% colnames(collect_metrics(x))] cli::cli_abort("Could not sort results by {.var {var_nm}}.") } - res %>% + summary_res %>% dplyr::slice(1) %>% dplyr::select(dplyr::all_of(param_names), .config) } -middle_eval_time <- function(x) { - x <- x[!is.na(x)] - times <- unique(x) - med_time <- median(x, na.rm = TRUE) - ind <- which.min(abs(times - med_time)) - eval_time <- times[ind] - eval_time -} - -# NOTE this chooses the time and subsets the data; break it up to only select -# time -choose_eval_time <- function(x, object, eval_time) { - mtrs <- .get_tune_metrics(object) - mtrs <- tibble::as_tibble(mtrs) - actual_metrics <- unique(x$.metric) - mtrs <- mtrs[mtrs$metric %in% actual_metrics, ] - - # Dynamic and integrated metrics need eval time as an input _but_ - # only dynamic metrics need them as outputs. So if the metric like - # `brier_survival_integrated()` is used, the evaluation time doesn't need - # to be specified for computations that use the metrics. - if (!any(mtrs$class == "dynamic_survival_metric")) { - return(x) - } - # TODO maybe issue a warning if there are missing values - if (is.null(eval_time)) { - eval_time <- middle_eval_time(x$.eval_time) - msg <- cli::pluralize("No evaluation time was set; a value of {eval_time} was used.") - rlang::warn(msg) - } else { - if (length(eval_time) > 1) { - rlang::abort("Please pick a single evaluation time point.") - } - times <- unique(x$.eval_time) - if (!any(times == eval_time)) { - msg <- cli::pluralize("No evaluation times matched a value of {eval_time}.") - rlang::abort(msg) - } +check_select_dots <- function(..., call = rlang::caller_env()) { + dots <- rlang::enquos(...) + if (length(dots) == 0) { + cli::cli_abort("Please choose at least one tuning parameter to sort in {.code ...}.", + call = call) } - x <- x[x$.eval_time == eval_time, ] - x + invisible(NULL) } - - diff --git a/inst/test_objects.R b/inst/test_objects.R index 1ca651dff..a57517143 100644 --- a/inst/test_objects.R +++ b/inst/test_objects.R @@ -1,7 +1,9 @@ library(tidymodels) library(scales) +library(censored) library(sessioninfo) library(testthat) +# also will require prodlim, mboost, kknn, and kernlab # ------------------------------------------------------------------------------ # "mt_*" test objects used in test-predictions.R, test-extract.R, and test-autoplot.R @@ -380,6 +382,51 @@ saveRDS( compress = "xz" ) +# ------------------------------------------------------------------------------ +# A single survival model + +set.seed(1) +sim_dat <- prodlim::SimSurv(200) %>% + mutate(event_time = Surv(time, event)) %>% + select(event_time, X1, X2) + +set.seed(2) +sim_rs <- vfold_cv(sim_dat) + +time_points <- c(10, 1, 5, 15) + +boost_spec <- + boost_tree(trees = tune()) %>% + set_mode("censored regression") %>% + set_engine("mboost") + +srv_mtr <- + metric_set( + brier_survival, + roc_auc_survival, + brier_survival_integrated, + concordance_survival + ) + +set.seed(2193) +surv_boost_tree_res <- + boost_spec %>% + tune_grid( + event_time ~ X1 + X2, + resamples = sim_rs, + grid = tibble(trees = c(1, 5, 10, 20, 100)), + metrics = srv_mtr, + eval_time = time_points + ) + +saveRDS( + surv_boost_tree_res, + file = testthat::test_path("data", "surv_boost_tree_res.rds"), + version = 2, + compress = "xz" +) + + # ------------------------------------------------------------------------------ sessioninfo::session_info() diff --git a/inst/test_objects.Rout b/inst/test_objects.Rout index 9e681743a..a0d8e916f 100644 --- a/inst/test_objects.Rout +++ b/inst/test_objects.Rout @@ -1,7 +1,7 @@ -R version 4.0.2 (2020-06-22) -- "Taking Off Again" -Copyright (C) 2020 The R Foundation for Statistical Computing -Platform: x86_64-apple-darwin17.0 (64-bit) +R version 4.3.1 (2023-06-16) -- "Beagle Scouts" +Copyright (C) 2023 The R Foundation for Statistical Computing +Platform: x86_64-apple-darwin20 (64-bit) R is free software and comes with ABSOLUTELY NO WARRANTY. You are welcome to redistribute it under certain conditions. @@ -18,20 +18,24 @@ Type 'demo()' for some demos, 'help()' for on-line help, or Type 'q()' to quit R. > library(tidymodels) -── Attaching packages ────────────────────────────────────── tidymodels 0.1.1 ── -✔ broom 0.7.0 ✔ recipes 0.1.13 -✔ dials 0.0.9.9000 ✔ rsample 0.0.8.9000 -✔ dplyr 1.0.2 ✔ tibble 3.0.3 -✔ ggplot2 3.3.2 ✔ tidyr 1.1.2 -✔ infer 0.5.3 ✔ tune 0.1.1.9000 -✔ modeldata 0.0.2 ✔ workflows 0.2.0.9000 -✔ parsnip 0.1.3 ✔ yardstick 0.0.7 -✔ purrr 0.3.4 +── Attaching packages ───────────────────────────────── tidymodels 1.1.1.9000 ── +✔ broom 1.0.5 ✔ recipes 1.0.8.9000 +✔ dials 1.2.0.9000 ✔ rsample 1.2.0.9000 +✔ dplyr 1.1.3 ✔ tibble 3.2.1 +✔ ggplot2 3.4.4 ✔ tidyr 1.3.0 +✔ infer 1.0.5 ✔ tune 1.1.2.9002 +✔ modeldata 1.2.0.9000 ✔ workflows 1.1.3.9000 +✔ parsnip 1.1.1.9002 ✔ workflowsets 1.0.1 +✔ purrr 1.0.2 ✔ yardstick 1.2.0.9001 ── Conflicts ───────────────────────────────────────── tidymodels_conflicts() ── ✖ purrr::discard() masks scales::discard() ✖ dplyr::filter() masks stats::filter() ✖ dplyr::lag() masks stats::lag() ✖ recipes::step() masks stats::step() +• Use tidymodels_prefer() to resolve common conflicts. +> library(scales) +> library(censored) +Loading required package: survival > library(sessioninfo) > library(testthat) @@ -41,6 +45,10 @@ The following object is masked from ‘package:tidyr’: matches +The following object is masked from ‘package:rsample’: + + matches + The following object is masked from ‘package:purrr’: is_null @@ -49,6 +57,7 @@ The following object is masked from ‘package:dplyr’: matches +> # also will require prodlim, mboost, kknn, and kernlab > > # ------------------------------------------------------------------------------ > # "mt_*" test objects used in test-predictions.R, test-extract.R, and test-autoplot.R @@ -77,7 +86,7 @@ The following object is masked from ‘package:dplyr’: > > get_coefs <- function(x) { + x %>% -+ pull_workflow_fit() %>% ++ extract_fit_parsnip() %>% + tidy() + } > @@ -109,31 +118,9 @@ The following object is masked from ‘package:dplyr’: + tune_grid(mt_spln_lm, + resamples = folds, + control = g_ctrl) -! Fold1: recipe 9/10: 'df' was too small; have used 3 -! Fold2: recipe 1/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold2: recipe 2/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold2: recipe 3/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold2: recipe 4/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold2: recipe 5/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold2: recipe 6/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold2: recipe 7/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold2: recipe 8/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold2: recipe 9/10: 'df' was too small; have used 3 -! Fold2: recipe 9/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold2: recipe 10/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold3: recipe 9/10: 'df' was too small; have used 3 -! Fold4: recipe 9/10: 'df' was too small; have used 3 -! Fold5: recipe 1/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold5: recipe 2/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold5: recipe 3/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold5: recipe 4/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold5: recipe 5/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold5: recipe 6/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold5: recipe 7/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold5: recipe 8/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold5: recipe 9/10: 'df' was too small; have used 3 -! Fold5: recipe 9/10, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold5: recipe 10/10, model 1/1 (predictions): some 'x' values beyond boundary k... +→ A | warning: prediction from rank-deficient fit; consider predict(., rankdeficient="NA") + There were issues with some computations A: x1 → B | warning: some 'x' values beyond boundary knots may cause ill-conditioned bases +There were issues with some computations A: x1 There were issues with some computations A: x1 B: x2 There were issues with some computations A: x1 B: x4 There were issues with some computations A: x1 B: x6 There were issues with some computations A: x1 B: x8 There were issues with some computations A: x2 B: x10 There were issues with some computations A: x2 B: x11 There were issues with some computations A: x2 B: x13 There were issues with some computations A: x2 B: x15 There were issues with some computations A: x2 B: x17 There were issues with some computations A: x2 B: x19 There were issues with some computations A: x2 B: x20 > > set.seed(8825) > mt_spln_lm_bo <- @@ -143,32 +130,8 @@ The following object is masked from ‘package:dplyr’: + iter = 3, + control = b_ctrl + ) -! Fold1: recipe 1/5: 'df' was too small; have used 3 -! Fold2: recipe 1/5: 'df' was too small; have used 3 -! Fold2: recipe 1/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold2: recipe 2/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold2: recipe 3/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold2: recipe 4/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold2: recipe 5/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold3: recipe 1/5: 'df' was too small; have used 3 -! Fold4: recipe 1/5: 'df' was too small; have used 3 -! Fold5: recipe 1/5: 'df' was too small; have used 3 -! Fold5: recipe 1/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 2/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 3/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 4/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 5/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold1: recipe 1/1: 'df' was too small; have used 3 -! Fold2: recipe 1/1: 'df' was too small; have used 3 -! Fold2: recipe 1/1, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold3: recipe 1/1: 'df' was too small; have used 3 -! Fold4: recipe 1/1: 'df' was too small; have used 3 -! Fold5: recipe 1/1: 'df' was too small; have used 3 -! Fold5: recipe 1/1, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold2: recipe 1/1, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 1/1, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold2: recipe 1/1, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 1/1, model 1/1 (predictions): some 'x' values beyond boundary kno... +→ A | warning: some 'x' values beyond boundary knots may cause ill-conditioned bases + There were issues with some computations A: x1 There were issues with some computations A: x2 There were issues with some computations A: x5 There were issues with some computations A: x6 There were issues with some computations A: x8 There were issues with some computations A: x10 There were issues with some computations A: x11 There were issues with some computations A: x12 There were issues with some computations A: x13 There were issues with some computations A: x14 There were issues with some computations A: x15 There were issues with some computations A: x16 There were issues with some computations A: x16 > > # ------------------------------------------------------------------------------ > @@ -177,20 +140,12 @@ The following object is masked from ‘package:dplyr’: + tune_grid( + mt_spln_knn, + resamples = folds, -+ grid = grid_regular(parameters(mt_spln_knn)), ++ grid = grid_regular(extract_parameter_set_dials(mt_spln_knn)), + control = g_ctrl + ) -! Fold1: recipe 1/3: 'df' was too small; have used 3 -! Fold2: recipe 1/3: 'df' was too small; have used 3 -! Fold2: recipe 1/3, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold2: recipe 2/3, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold2: recipe 3/3, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold3: recipe 1/3: 'df' was too small; have used 3 -! Fold4: recipe 1/3: 'df' was too small; have used 3 -! Fold5: recipe 1/3: 'df' was too small; have used 3 -! Fold5: recipe 1/3, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 2/3, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 3/3, model 1/1 (predictions): some 'x' values beyond boundary kno... +→ A | error: No tidy method for objects of class train.kknn + There were issues with some computations A: x1 There were issues with some computations A: x2 There were issues with some computations A: x4 → B | warning: some 'x' values beyond boundary knots may cause ill-conditioned bases +There were issues with some computations A: x4 There were issues with some computations A: x5 B: x1 There were issues with some computations A: x6 B: x3 There were issues with some computations A: x8 B: x3 There were issues with some computations A: x9 B: x3 There were issues with some computations A: x11 B: x3 There were issues with some computations A: x13 B: x3 There were issues with some computations A: x13 B: x4 There were issues with some computations A: x14 B: x4 There were issues with some computations A: x15 B: x6 There were issues with some computations A: x15 B: x6 > > set.seed(8825) > mt_spln_knn_bo <- @@ -198,27 +153,9 @@ The following object is masked from ‘package:dplyr’: + resamples = folds, + iter = 3, + control = b_ctrl) -! Fold2: recipe 1/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold2: recipe 2/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold2: recipe 3/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold2: recipe 4/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold2: recipe 5/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 1/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 2/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 3/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 4/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 5/5, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold1: recipe 1/1: 'df' was too small; have used 3 -! Fold2: recipe 1/1: 'df' was too small; have used 3 -! Fold2: recipe 1/1, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold3: recipe 1/1: 'df' was too small; have used 3 -! Fold4: recipe 1/1: 'df' was too small; have used 3 -! Fold5: recipe 1/1: 'df' was too small; have used 3 -! Fold5: recipe 1/1, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold2: recipe 1/1, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 1/1, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold2: recipe 1/1, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 1/1, model 1/1 (predictions): some 'x' values beyond boundary kno... +→ A | error: No tidy method for objects of class train.kknn + There were issues with some computations A: x1 There were issues with some computations A: x2 There were issues with some computations A: x4 There were issues with some computations A: x6 → B | warning: some 'x' values beyond boundary knots may cause ill-conditioned bases +There were issues with some computations A: x6 There were issues with some computations A: x8 B: x2 There were issues with some computations A: x9 B: x4 There were issues with some computations A: x11 B: x5 There were issues with some computations A: x13 B: x5 There were issues with some computations A: x15 B: x5 There were issues with some computations A: x17 B: x5 There were issues with some computations A: x19 B: x5 There were issues with some computations A: x21 B: x5 There were issues with some computations A: x23 B: x7 There were issues with some computations A: x25 B: x9 There were issues with some computations A: x26 B: x10 There were issues with some computations A: x28 B: x11 There were issues with some computations A: x30 B: x11 There were issues with some computations A: x31 B: x12 There were issues with some computations A: x32 B: x12 There were issues with some computations A: x34 B: x13 There were issues with some computations A: x36 B: x14 There were issues with some computations A: x38 B: x15 There were issues with some computations A: x40 B: x15 There were issues with some computations A: x40 B: x16 > > set.seed(8825) > mt_spln_knn_bo_sep <- @@ -227,44 +164,23 @@ The following object is masked from ‘package:dplyr’: + resamples = folds, + iter = 3, + control = b_ctrl) -! Fold1: recipe 1/4: 'df' was too small; have used 3 -! Fold2: recipe 1/4: 'df' was too small; have used 3 -! Fold2: recipe 1/4, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold2: recipe 2/4, model 1/2 (predictions): some 'x' values beyond boundary kno... -! Fold2: recipe 2/4, model 2/2 (predictions): some 'x' values beyond boundary kno... -! Fold2: recipe 3/4, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold2: recipe 4/4, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold3: recipe 1/4: 'df' was too small; have used 3 -! Fold4: recipe 1/4: 'df' was too small; have used 3 -! Fold5: recipe 1/4: 'df' was too small; have used 3 -! Fold5: recipe 1/4, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 2/4, model 1/2 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 2/4, model 2/2 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 3/4, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 4/4, model 1/1 (predictions): some 'x' values beyond boundary kno... -! The Gaussian process model is being fit using 12 features but only has 5 +→ A | error: No tidy method for objects of class train.kknn + There were issues with some computations A: x1 There were issues with some computations A: x2 There were issues with some computations A: x5 → B | warning: some 'x' values beyond boundary knots may cause ill-conditioned bases +There were issues with some computations A: x5 There were issues with some computations A: x6 B: x1 There were issues with some computations A: x8 B: x3 There were issues with some computations A: x10 B: x4 There were issues with some computations A: x12 B: x5 There were issues with some computations A: x14 B: x5 There were issues with some computations A: x16 B: x5 There were issues with some computations A: x19 B: x5 There were issues with some computations A: x21 B: x5 There were issues with some computations A: x22 B: x6 There were issues with some computations A: x24 B: x8 ! The Gaussian process model is being fit using 12 features but only has 5 data points to do so. This may cause errors or a poor model fit. -! Gaussian process model: did not converge in 10 iterations -! Fold2: recipe 1/1, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 1/1, model 1/1 (predictions): some 'x' values beyond boundary kno... -! The Gaussian process model is being fit using 12 features but only has 6 + → C | warning: did not converge in 10 iterations +There were issues with some computations A: x24 B: x8 There were issues with some computations A: x25 B: x10 C: x1 There were issues with some computations A: x26 B: x10 C: x1 There were issues with some computations A: x27 B: x10 C: x1 There were issues with some computations A: x29 B: x11 C: x1 ! The Gaussian process model is being fit using 12 features but only has 6 data points to do so. This may cause errors or a poor model fit. -! Fold2: recipe 1/1, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold5: recipe 1/1, model 1/1 (predictions): some 'x' values beyond boundary kno... -! The Gaussian process model is being fit using 12 features but only has 7 + There were issues with some computations A: x31 B: x12 C: x1 There were issues with some computations A: x33 B: x13 C: x1 There were issues with some computations A: x35 B: x13 C: x1 ! The Gaussian process model is being fit using 12 features but only has 7 data points to do so. This may cause errors or a poor model fit. -! Fold1: recipe 1/1: 'df' was too small; have used 3 -! Fold2: recipe 1/1: 'df' was too small; have used 3 -! Fold2: recipe 1/1, model 1/1 (predictions): some 'x' values beyond boundary kno... -! Fold3: recipe 1/1: 'df' was too small; have used 3 -! Fold4: recipe 1/1: 'df' was too small; have used 3 -! Fold5: recipe 1/1: 'df' was too small; have used 3 -! Fold5: recipe 1/1, model 1/1 (predictions): some 'x' values beyond boundary kno... + There were issues with some computations A: x36 B: x14 C: x1 There were issues with some computations A: x38 B: x15 C: x1 There were issues with some computations A: x40 B: x15 C: x1 There were issues with some computations A: x40 B: x16 C: x1 > > # ------------------------------------------------------------------------------ > > set.seed(8825) > mt_knn_grid <- tune_grid(mt_knn, resamples = folds, control = g_ctrl) +→ A | error: No tidy method for objects of class train.kknn + There were issues with some computations A: x1 There were issues with some computations A: x2 There were issues with some computations A: x4 There were issues with some computations A: x5 There were issues with some computations A: x5 > > set.seed(8825) > mt_knn_bo <- @@ -272,12 +188,14 @@ The following object is masked from ‘package:dplyr’: + resamples = folds, + iter = 3, + control = b_ctrl) +→ A | error: No tidy method for objects of class train.kknn + There were issues with some computations A: x1 There were issues with some computations A: x3 There were issues with some computations A: x5 There were issues with some computations A: x6 There were issues with some computations A: x9 There were issues with some computations A: x11 There were issues with some computations A: x14 There were issues with some computations A: x16 There were issues with some computations A: x19 There were issues with some computations A: x20 > > # ------------------------------------------------------------------------------ > > save( + list = grep("^mt_", ls(), value = TRUE), -+ file = test_path("test_objects.RData"), ++ file = test_path("data", "test_objects.RData"), + version = 2, + compress = "xz" + ) @@ -309,7 +227,7 @@ The following object is masked from ‘package:dplyr’: + add_model(knn_model) > > two_class_set <- -+ parameters(two_class_wflow) %>% ++ extract_parameter_set_dials(two_class_wflow) %>% + update(K = neighbors(c(1, 50))) %>% + update(exponent = dist_power(c(1 / 10, 2))) > @@ -342,35 +260,35 @@ The following object is masked from ‘package:dplyr’: > > saveRDS( + knn_results, -+ file = testthat::test_path("knn_results.rds"), ++ file = testthat::test_path("data", "knn_results.rds"), + version = 2, + compress = "xz" + ) > > saveRDS( + two_class_set, -+ file = testthat::test_path("knn_set.rds"), ++ file = testthat::test_path("data", "knn_set.rds"), + version = 2, + compress = "xz" + ) > > saveRDS( + two_class_grid, -+ file = testthat::test_path("knn_grid.rds"), ++ file = testthat::test_path("data", "knn_grid.rds"), + version = 2, + compress = "xz" + ) > > saveRDS( + knn_set, -+ file = testthat::test_path("knn_set.rds"), ++ file = testthat::test_path("data", "knn_set.rds"), + version = 2, + compress = "xz" + ) > > saveRDS( + knn_gp, -+ file = testthat::test_path("knn_gp.rds"), ++ file = testthat::test_path("data", "knn_gp.rds"), + version = 2, + compress = "xz" + ) @@ -393,7 +311,7 @@ The following object is masked from ‘package:dplyr’: + add_model(svm_model) > > two_class_set <- -+ parameters(two_class_wflow) %>% ++ extract_parameter_set_dials(two_class_wflow) %>% + update(cost = cost(c(-10, 4))) > > set.seed(2494) @@ -411,25 +329,10 @@ The following object is masked from ‘package:dplyr’: + metrics = class_only, + control = control_grid(save_pred = TRUE) + ) - -Attaching package: ‘kernlab’ - -The following object is masked from ‘package:purrr’: - - cross - -The following object is masked from ‘package:ggplot2’: - - alpha - -The following object is masked from ‘package:scales’: - - alpha - > > saveRDS( + svm_results, -+ file = testthat::test_path("svm_results.rds"), ++ file = testthat::test_path("data", "svm_results.rds"), + version = 2, + compress = "xz" + ) @@ -449,7 +352,7 @@ The following object is masked from ‘package:scales’: > > saveRDS( + svm_reg_results, -+ file = testthat::test_path("svm_reg_results.rds"), ++ file = testthat::test_path("data", "svm_reg_results.rds"), + version = 2, + compress = "xz" + ) @@ -487,6 +390,9 @@ The following object is masked from ‘package:scales’: + update(deg_free = deg_free(c(2, 10))) %>% + update(`wt degree` = degree_int(1:2)) %>% + update(`wt df` = deg_free(c(2, 10))) +Warning message: +`parameters.workflow()` was deprecated in tune 0.1.6.9003. +ℹ Please use `hardhat::extract_parameter_set_dials()` instead. > > set.seed(255) > cars_grid <- @@ -501,306 +407,14 @@ The following object is masked from ‘package:scales’: + grid = cars_grid, + control = control_grid(verbose = FALSE, save_pred = TRUE) + ) -! Fold01, Repeat1: recipe 15/36, model 1/1 (predictions): prediction from a rank-deficient ... -! Fold01, Repeat1: recipe 18/36, model 1/1 (predictions): prediction from a rank-deficient ... -! Fold01, Repeat1: recipe 33/36, model 1/1 (predictions): prediction from a rank-deficient ... -! Fold01, Repeat1: recipe 36/36, model 1/1 (predictions): prediction from a rank-deficient ... -! Fold02, Repeat1: recipe 1/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 2/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 3/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 4/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 5/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 6/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 7/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 8/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 9/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 10/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 11/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 12/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 13/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 14/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 15/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 16/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 17/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 18/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 19/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 20/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 21/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 22/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 23/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 24/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 25/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 26/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 27/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 28/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 29/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 30/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 31/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 32/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 33/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 34/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 35/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat1: recipe 36/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 1/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 2/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 3/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 4/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 5/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 6/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 7/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 8/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 9/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 10/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 11/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 12/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 13/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 14/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 15/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 16/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 17/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 18/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 19/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 20/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 21/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 22/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 23/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 24/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 25/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 26/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 27/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 28/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 29/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 30/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 31/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 32/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 33/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 34/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 35/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold07, Repeat1: recipe 36/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 1/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 2/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 3/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 4/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 5/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 6/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 7/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 8/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 9/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 10/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 11/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 12/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 13/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 14/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 15/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 16/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 17/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 18/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 19/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 20/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 21/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 22/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 23/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 24/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 25/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 26/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 27/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 28/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 29/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 30/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 31/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 32/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 33/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 34/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 35/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat1: recipe 36/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 1/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 2/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 3/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 4/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 5/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 6/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 7/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 8/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 9/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 10/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 11/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 12/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 13/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 14/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 15/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 16/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 17/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 18/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 19/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 20/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 21/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 22/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 23/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 24/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 25/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 26/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 27/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 28/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 29/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 30/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 31/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 32/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 33/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 34/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 35/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat1: recipe 36/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 1/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 2/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 3/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 4/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 5/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 6/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 7/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 8/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 9/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 10/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 11/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 12/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 13/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 14/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 15/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 16/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 17/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 18/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 19/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 20/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 21/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 22/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 23/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 24/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 25/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 26/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 27/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 28/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 29/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 30/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 31/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 32/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 33/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 34/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 35/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold01, Repeat2: recipe 36/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold02, Repeat2: recipe 15/36, model 1/1 (predictions): prediction from a rank-deficient ... -! Fold02, Repeat2: recipe 18/36, model 1/1 (predictions): prediction from a rank-deficient ... -! Fold02, Repeat2: recipe 33/36, model 1/1 (predictions): prediction from a rank-deficient ... -! Fold02, Repeat2: recipe 36/36, model 1/1 (predictions): prediction from a rank-deficient ... -! Fold06, Repeat2: recipe 1/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 2/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 3/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 4/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 5/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 6/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 7/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 8/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 9/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 10/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 11/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 12/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 13/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 14/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 15/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 16/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 17/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 18/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 19/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 20/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 21/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 22/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 23/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 24/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 25/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 26/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 27/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 28/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 29/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 30/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 31/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 32/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 33/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 34/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 35/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold06, Repeat2: recipe 36/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 1/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 2/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 3/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 4/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 5/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 6/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 7/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 8/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 9/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 10/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 11/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 12/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 13/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 14/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 15/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 16/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 17/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 18/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 19/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 20/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 21/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 22/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 23/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 24/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 25/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 26/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 27/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 28/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 29/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 30/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 31/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 32/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 33/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 34/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 35/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold08, Repeat2: recipe 36/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 1/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 2/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 3/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 4/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 5/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 6/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 7/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 8/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 9/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 10/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 11/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 12/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 13/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 14/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 15/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 16/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 17/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 18/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 19/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 20/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 21/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 22/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 23/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 24/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 25/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 26/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 27/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 28/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 29/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 30/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 31/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 32/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 33/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 34/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 35/36, model 1/1 (predictions): some 'x' values beyond boundary k... -! Fold10, Repeat2: recipe 36/36, model 1/1 (predictions): some 'x' values beyond boundary k... +→ A | warning: prediction from rank-deficient fit; consider predict(., rankdeficient="NA") + There were issues with some computations A: x1 There were issues with some computations A: x2 There were issues with some computations A: x3 There were issues with some computations A: x4 → B | warning: some 'x' values beyond boundary knots may cause ill-conditioned bases +There were issues with some computations A: x4 There were issues with some computations A: x4 B: x2 There were issues with some computations A: x4 B: x4 There were issues with some computations A: x4 B: x6 There were issues with some computations A: x4 B: x8 There were issues with some computations A: x4 B: x10 There were issues with some computations A: x4 B: x11 There were issues with some computations A: x4 B: x13 → C | warning: some 'x' values beyond boundary knots may cause ill-conditioned bases, prediction from rank-deficient fit; consider predict(., rankdeficient="NA") +There were issues with some computations A: x4 B: x13 There were issues with some computations A: x4 B: x14 C: x1 There were issues with some computations A: x4 B: x16 C: x1 There were issues with some computations A: x4 B: x16 C: x2 There were issues with some computations A: x4 B: x18 C: x2 There were issues with some computations A: x4 B: x20 C: x2 There were issues with some computations A: x4 B: x22 C: x2 There were issues with some computations A: x4 B: x24 C: x2 There were issues with some computations A: x4 B: x26 C: x2 There were issues with some computations A: x4 B: x28 C: x2 There were issues with some computations A: x4 B: x29 C: x2 There were issues with some computations A: x4 B: x30 C: x3 There were issues with some computations A: x4 B: x32 C: x3 There were issues with some computations A: x4 B: x33 C: x4 There were issues with some computations A: x4 B: x35 C: x4 There were issues with some computations A: x4 B: x36 C: x4 There were issues with some computations A: x4 B: x38 C: x4 There were issues with some computations A: x4 B: x40 C: x4 There were issues with some computations A: x4 B: x42 C: x4 There were issues with some computations A: x4 B: x44 C: x4 There were issues with some computations A: x4 B: x46 C: x4 There were issues with some computations A: x4 B: x47 C: x4 There were issues with some computations A: x4 B: x49 C: x4 There were issues with some computations A: x4 B: x51 C: x4 There were issues with some computations A: x4 B: x53 C: x4 There were issues with some computations A: x4 B: x55 C: x4 There were issues with some computations A: x4 B: x57 C: x4 There were issues with some computations A: x4 B: x58 C: x4 There were issues with some computations A: x4 B: x60 C: x4 There were issues with some computations A: x4 B: x62 C: x4 There were issues with some computations A: x4 B: x64 C: x4 There were issues with some computations A: x4 B: x66 C: x4 There were issues with some computations A: x4 B: x67 C: x4 There were issues with some computations A: x4 B: x69 C: x4 There were issues with some computations A: x4 B: x71 C: x4 There were issues with some computations A: x4 B: x73 C: x4 There were issues with some computations A: x4 B: x74 C: x4 There were issues with some computations A: x4 B: x76 C: x4 There were issues with some computations A: x4 B: x77 C: x4 There were issues with some computations A: x4 B: x79 C: x4 There were issues with some computations A: x4 B: x81 C: x4 There were issues with some computations A: x4 B: x82 C: x4 There were issues with some computations A: x4 B: x84 C: x4 There were issues with some computations A: x4 B: x86 C: x4 There were issues with some computations A: x4 B: x87 C: x4 There were issues with some computations A: x4 B: x89 C: x4 There were issues with some computations A: x4 B: x90 C: x4 There were issues with some computations A: x4 B: x92 C: x4 There were issues with some computations A: x4 B: x94 C: x4 There were issues with some computations A: x4 B: x95 C: x4 There were issues with some computations A: x4 B: x97 C: x4 There were issues with some computations A: x4 B: x99 C: x4 There were issues with some computations A: x4 B: x101 C: x4 There were issues with some computations A: x4 B: x102 C: x4 There were issues with some computations A: x4 B: x104 C: x4 There were issues with some computations A: x4 B: x105 C: x4 There were issues with some computations A: x4 B: x107 C: x4 There were issues with some computations A: x4 B: x108 C: x4 There were issues with some computations A: x4 B: x110 C: x4 There were issues with some computations A: x4 B: x112 C: x4 There were issues with some computations A: x4 B: x114 C: x4 There were issues with some computations A: x4 B: x116 C: x4 There were issues with some computations A: x4 B: x117 C: x4 There were issues with some computations A: x4 B: x119 C: x4 There were issues with some computations A: x4 B: x121 C: x4 There were issues with some computations A: x4 B: x123 C: x4 There were issues with some computations A: x4 B: x124 C: x4 There were issues with some computations A: x4 B: x125 C: x4 There were issues with some computations A: x4 B: x126 C: x4 There were issues with some computations A: x4 B: x128 C: x4 There were issues with some computations A: x4 B: x130 C: x4 There were issues with some computations A: x4 B: x132 C: x4 There were issues with some computations A: x4 B: x134 C: x4 There were issues with some computations A: x4 B: x136 C: x4 There were issues with some computations A: x4 B: x138 C: x4 There were issues with some computations A: x4 B: x140 C: x4 There were issues with some computations A: x4 B: x141 C: x4 There were issues with some computations A: x4 B: x143 C: x4 There were issues with some computations A: x4 B: x145 C: x4 There were issues with some computations A: x4 B: x147 C: x4 There were issues with some computations A: x4 B: x149 C: x4 There were issues with some computations A: x4 B: x150 C: x4 There were issues with some computations A: x4 B: x152 C: x4 There were issues with some computations A: x4 B: x154 C: x4 There were issues with some computations A: x4 B: x155 C: x5 There were issues with some computations A: x4 B: x156 C: x6 There were issues with some computations A: x4 B: x157 C: x6 There were issues with some computations A: x4 B: x159 C: x6 There were issues with some computations A: x4 B: x161 C: x6 There were issues with some computations A: x4 B: x163 C: x6 There were issues with some computations A: x4 B: x165 C: x6 There were issues with some computations A: x4 B: x167 C: x6 There were issues with some computations A: x4 B: x169 C: x6 There were issues with some computations A: x4 B: x170 C: x6 There were issues with some computations A: x4 B: x171 C: x7 There were issues with some computations A: x4 B: x172 C: x8 There were issues with some computations A: x5 B: x172 C: x8 There were issues with some computations A: x6 B: x172 C: x8 There were issues with some computations A: x7 B: x172 C: x8 There were issues with some computations A: x8 B: x172 C: x8 There were issues with some computations A: x8 B: x173 C: x8 There were issues with some computations A: x8 B: x174 C: x8 There were issues with some computations A: x8 B: x175 C: x8 There were issues with some computations A: x8 B: x177 C: x8 There were issues with some computations A: x8 B: x179 C: x8 There were issues with some computations A: x8 B: x180 C: x8 There were issues with some computations A: x8 B: x182 C: x8 There were issues with some computations A: x8 B: x184 C: x8 There were issues with some computations A: x8 B: x186 C: x8 There were issues with some computations A: x8 B: x187 C: x8 There were issues with some computations A: x8 B: x189 C: x8 There were issues with some computations A: x8 B: x190 C: x8 There were issues with some computations A: x8 B: x191 C: x8 There were issues with some computations A: x8 B: x193 C: x8 There were issues with some computations A: x8 B: x195 C: x8 There were issues with some computations A: x8 B: x196 C: x8 There were issues with some computations A: x8 B: x198 C: x8 There were issues with some computations A: x8 B: x199 C: x8 There were issues with some computations A: x8 B: x201 C: x8 There were issues with some computations A: x8 B: x203 C: x8 There were issues with some computations A: x8 B: x204 C: x8 There were issues with some computations A: x8 B: x206 C: x8 There were issues with some computations A: x8 B: x208 C: x8 There were issues with some computations A: x8 B: x209 C: x8 There were issues with some computations A: x8 B: x211 C: x8 There were issues with some computations A: x8 B: x213 C: x8 There were issues with some computations A: x8 B: x214 C: x8 There were issues with some computations A: x8 B: x216 C: x8 There were issues with some computations A: x8 B: x217 C: x8 There were issues with some computations A: x8 B: x219 C: x8 There were issues with some computations A: x8 B: x221 C: x8 There were issues with some computations A: x8 B: x222 C: x8 There were issues with some computations A: x8 B: x224 C: x8 There were issues with some computations A: x8 B: x225 C: x8 There were issues with some computations A: x8 B: x227 C: x8 There were issues with some computations A: x8 B: x229 C: x8 There were issues with some computations A: x8 B: x230 C: x8 There were issues with some computations A: x8 B: x232 C: x8 There were issues with some computations A: x8 B: x234 C: x8 There were issues with some computations A: x8 B: x235 C: x8 There were issues with some computations A: x8 B: x237 C: x8 There were issues with some computations A: x8 B: x238 C: x8 There were issues with some computations A: x8 B: x240 C: x8 There were issues with some computations A: x8 B: x242 C: x8 There were issues with some computations A: x8 B: x243 C: x8 There were issues with some computations A: x8 B: x245 C: x8 There were issues with some computations A: x8 B: x246 C: x8 There were issues with some computations A: x8 B: x248 C: x8 There were issues with some computations A: x8 B: x250 C: x8 There were issues with some computations A: x8 B: x251 C: x8 There were issues with some computations A: x8 B: x253 C: x8 There were issues with some computations A: x8 B: x255 C: x8 There were issues with some computations A: x8 B: x256 C: x8 There were issues with some computations A: x8 B: x258 C: x8 There were issues with some computations A: x8 B: x260 C: x8 There were issues with some computations A: x8 B: x261 C: x8 There were issues with some computations A: x8 B: x263 C: x8 There were issues with some computations A: x8 B: x265 C: x8 There were issues with some computations A: x8 B: x267 C: x8 There were issues with some computations A: x8 B: x268 C: x8 There were issues with some computations A: x8 B: x270 C: x8 There were issues with some computations A: x8 B: x272 C: x8 There were issues with some computations A: x8 B: x274 C: x8 There were issues with some computations A: x8 B: x275 C: x8 There were issues with some computations A: x8 B: x277 C: x8 There were issues with some computations A: x8 B: x278 C: x8 There were issues with some computations A: x8 B: x280 C: x8 There were issues with some computations A: x8 B: x280 C: x8 > > saveRDS( + rcv_results, -+ file = testthat::test_path("rcv_results.rds"), ++ file = testthat::test_path("data", "rcv_results.rds"), + version = 2, + compress = "xz" + ) @@ -823,16 +437,16 @@ The following object is masked from ‘package:scales’: > lm_resamples # Resampling results # 3-fold cross-validation -# A tibble: 3 x 4 +# A tibble: 3 × 4 splits id .metrics .notes -1 Fold1 -2 Fold2 -3 Fold3 +1 Fold1 +2 Fold2 +3 Fold3 > > saveRDS( + lm_resamples, -+ file = testthat::test_path("lm_resamples.rds"), ++ file = testthat::test_path("data", "lm_resamples.rds"), + version = 2, + compress = "xz" + ) @@ -856,113 +470,174 @@ The following object is masked from ‘package:scales’: > > set.seed(2934) > lm_bayes <- tune_bayes(wflow, folds, initial = 4, iter = 3) -! Fold1: recipe 3/4, model 1/1 (predictions): prediction from a rank-deficient fi... -! Fold1: recipe 4/4, model 1/1 (predictions): prediction from a rank-deficient fi... -! Fold2: recipe 3/4, model 1/1 (predictions): prediction from a rank-deficient fi... -! Fold2: recipe 4/4, model 1/1 (predictions): prediction from a rank-deficient fi... +→ A | warning: prediction from rank-deficient fit; consider predict(., rankdeficient="NA") + There were issues with some computations A: x1 There were issues with some computations A: x2 There were issues with some computations A: x3 There were issues with some computations A: x5 There were issues with some computations A: x6 There were issues with some computations A: x6 > > saveRDS( + lm_bayes, -+ file = testthat::test_path("lm_bayes.rds"), ++ file = testthat::test_path("data", "lm_bayes.rds"), ++ version = 2, ++ compress = "xz" ++ ) +> +> # ------------------------------------------------------------------------------ +> # A single survival model +> +> set.seed(1) +> sim_dat <- prodlim::SimSurv(200) %>% ++ mutate(event_time = Surv(time, event)) %>% ++ select(event_time, X1, X2) +> +> set.seed(2) +> sim_rs <- vfold_cv(sim_dat) +> +> time_points <- c(10, 1, 5, 15) +> +> boost_spec <- ++ boost_tree(trees = tune()) %>% ++ set_mode("censored regression") %>% ++ set_engine("mboost") +> +> srv_mtr <- ++ metric_set( ++ brier_survival, ++ roc_auc_survival, ++ brier_survival_integrated, ++ concordance_survival ++ ) +> +> set.seed(2193) +> surv_boost_tree_res <- ++ boost_spec %>% ++ tune_grid( ++ event_time ~ X1 + X2, ++ resamples = sim_rs, ++ grid = tibble(trees = c(1, 5, 10, 20, 100)), ++ metrics = srv_mtr, ++ eval_time = time_points ++ ) +> +> saveRDS( ++ surv_boost_tree_res, ++ file = testthat::test_path("data", "surv_boost_tree_res.rds"), + version = 2, + compress = "xz" + ) > +> > # ------------------------------------------------------------------------------ > > sessioninfo::session_info() ─ Session info ─────────────────────────────────────────────────────────────── - setting value - version R version 4.0.2 (2020-06-22) - os macOS Mojave 10.14.6 - system x86_64, darwin17.0 - ui X11 - language (EN) - collate en_US.UTF-8 - ctype en_US.UTF-8 - tz America/Denver - date 2020-10-06 + setting value + version R version 4.3.1 (2023-06-16) + os macOS Monterey 12.7.1 + system x86_64, darwin20 + ui X11 + language (EN) + collate en_US.UTF-8 + ctype en_US.UTF-8 + tz America/New_York + date 2023-12-05 + pandoc 2.17.1.1 @ /usr/local/bin/pandoc ─ Packages ─────────────────────────────────────────────────────────────────── - package * version date lib source - assertthat 0.2.1 2019-03-21 [1] CRAN (R 4.0.0) - backports 1.1.10 2020-09-15 [1] CRAN (R 4.0.2) - broom * 0.7.0 2020-07-09 [1] CRAN (R 4.0.0) - class 7.3-17 2020-04-26 [1] CRAN (R 4.0.2) - cli 2.0.2 2020-02-28 [1] CRAN (R 4.0.0) - codetools 0.2-16 2018-12-24 [1] CRAN (R 4.0.2) - colorspace 1.4-1 2019-03-18 [1] CRAN (R 4.0.0) - crayon 1.3.4.9000 2020-09-19 [1] Github (r-lib/crayon@6b3f0c6) - dials * 0.0.9.9000 2020-10-05 [1] Github (tidymodels/dials@2b79300) - DiceDesign 1.8-1 2019-07-31 [1] CRAN (R 4.0.0) - digest 0.6.25 2020-02-23 [1] CRAN (R 4.0.0) - dplyr * 1.0.2 2020-08-18 [1] CRAN (R 4.0.2) - ellipsis 0.3.1 2020-05-15 [1] CRAN (R 4.0.0) - fansi 0.4.1 2020-01-08 [1] CRAN (R 4.0.0) - foreach 1.5.0 2020-03-30 [1] CRAN (R 4.0.0) - furrr 0.1.0 2018-05-16 [1] CRAN (R 4.0.0) - future 1.19.1 2020-09-22 [1] CRAN (R 4.0.2) - generics 0.0.2 2018-11-29 [1] CRAN (R 4.0.0) - ggplot2 * 3.3.2 2020-06-19 [1] CRAN (R 4.0.0) - globals 0.13.0 2020-09-17 [1] CRAN (R 4.0.2) - glue 1.4.2 2020-08-27 [1] CRAN (R 4.0.2) - gower 0.2.2 2020-06-23 [1] CRAN (R 4.0.0) - GPfit 1.0-8 2019-02-08 [1] CRAN (R 4.0.0) - gtable 0.3.0 2019-03-25 [1] CRAN (R 4.0.0) - hardhat 0.1.4 2020-07-02 [1] CRAN (R 4.0.2) - igraph 1.2.5 2020-03-19 [1] CRAN (R 4.0.0) - infer * 0.5.3 2020-07-14 [1] CRAN (R 4.0.0) - ipred 0.9-9 2019-04-28 [1] CRAN (R 4.0.0) - iterators 1.0.12 2019-07-26 [1] CRAN (R 4.0.0) - kernlab * 0.9-29 2019-11-12 [1] CRAN (R 4.0.0) - kknn * 1.3.1 2016-03-26 [1] CRAN (R 4.0.0) - lattice 0.20-41 2020-04-02 [1] CRAN (R 4.0.2) - lava 1.6.8 2020-09-26 [1] CRAN (R 4.0.2) - lhs 1.1.0 2020-09-29 [1] CRAN (R 4.0.2) - lifecycle 0.2.0 2020-03-06 [1] CRAN (R 4.0.0) - listenv 0.8.0 2019-12-05 [1] CRAN (R 4.0.0) - lubridate 1.7.9 2020-06-08 [1] CRAN (R 4.0.0) - magrittr 1.5.0.9000 2020-09-27 [1] Github (tidyverse/magrittr@0221e18) - MASS 7.3-53 2020-09-09 [1] CRAN (R 4.0.2) - Matrix 1.2-18 2019-11-27 [1] CRAN (R 4.0.2) - modeldata * 0.0.2 2020-06-22 [1] CRAN (R 4.0.0) - munsell 0.5.0 2018-06-12 [1] CRAN (R 4.0.0) - nnet 7.3-14 2020-04-26 [1] CRAN (R 4.0.2) - parsnip * 0.1.3 2020-08-04 [1] CRAN (R 4.0.2) - pillar 1.4.6 2020-07-10 [1] CRAN (R 4.0.0) - pkgconfig 2.0.3 2019-09-22 [1] CRAN (R 4.0.0) - plyr 1.8.6 2020-03-03 [1] CRAN (R 4.0.0) - pROC 1.16.2 2020-03-19 [1] CRAN (R 4.0.0) - prodlim 2019.11.13 2019-11-17 [1] CRAN (R 4.0.0) - purrr * 0.3.4 2020-04-17 [1] CRAN (R 4.0.0) - R6 2.4.1 2019-11-12 [1] CRAN (R 4.0.0) - Rcpp 1.0.5 2020-07-06 [1] CRAN (R 4.0.0) - recipes * 0.1.13 2020-06-23 [1] CRAN (R 4.0.2) - rlang 0.4.7 2020-07-09 [1] CRAN (R 4.0.0) - rpart 4.1-15 2019-04-12 [1] CRAN (R 4.0.2) - rsample * 0.0.8.9000 2020-10-05 [1] Github (tidymodels/rsample@ae99c86) - rstudioapi 0.11 2020-02-07 [1] CRAN (R 4.0.0) - scales * 1.1.1 2020-05-11 [1] CRAN (R 4.0.0) - sessioninfo * 1.1.1 2018-11-05 [1] CRAN (R 4.0.0) - survival 3.2-3 2020-06-13 [1] CRAN (R 4.0.1) - testthat * 2.3.2 2020-03-02 [1] CRAN (R 4.0.0) - tibble * 3.0.3 2020-07-10 [1] CRAN (R 4.0.2) - tidymodels * 0.1.1 2020-07-14 [1] CRAN (R 4.0.2) - tidyr * 1.1.2 2020-08-27 [1] CRAN (R 4.0.2) - tidyselect 1.1.0 2020-05-11 [1] CRAN (R 4.0.0) - timeDate 3043.102 2018-02-21 [1] CRAN (R 4.0.0) - tune * 0.1.1.9000 2020-10-06 [1] local - utf8 1.1.4 2018-05-24 [1] CRAN (R 4.0.0) - vctrs 0.3.4 2020-08-29 [1] CRAN (R 4.0.2) - withr 2.3.0 2020-09-22 [1] CRAN (R 4.0.2) - workflows * 0.2.0.9000 2020-10-05 [1] Github (tidymodels/workflows@40482ae) - yardstick * 0.0.7 2020-07-13 [1] CRAN (R 4.0.0) + package * version date (UTC) lib source + backports 1.4.1 2021-12-13 [1] CRAN (R 4.3.0) + brio 1.1.3 2021-11-30 [1] CRAN (R 4.3.0) + broom * 1.0.5 2023-06-09 [1] CRAN (R 4.3.0) + censored * 0.2.0.9000 2023-11-13 [1] Github (tidymodels/censored@f9eccb6) + class 7.3-22 2023-05-03 [2] CRAN (R 4.3.1) + cli 3.6.1 2023-03-23 [1] CRAN (R 4.3.0) + codetools 0.2-19 2023-02-01 [2] CRAN (R 4.3.1) + colorspace 2.1-0 2023-01-23 [1] CRAN (R 4.3.0) + data.table 1.14.8 2023-02-17 [1] CRAN (R 4.3.0) + dials * 1.2.0.9000 2023-11-13 [1] Github (tidymodels/dials@3e42198) + DiceDesign 1.9 2021-02-13 [1] CRAN (R 4.3.0) + digest 0.6.33 2023-07-07 [1] CRAN (R 4.3.0) + dplyr * 1.1.3 2023-09-03 [1] CRAN (R 4.3.0) + ellipsis 0.3.2 2021-04-29 [1] CRAN (R 4.3.0) + fansi 1.0.5 2023-10-08 [1] CRAN (R 4.3.0) + foreach 1.5.2 2022-02-02 [1] CRAN (R 4.3.0) + Formula 1.2-5 2023-02-24 [1] CRAN (R 4.3.0) + furrr 0.3.1 2022-08-15 [1] CRAN (R 4.3.0) + future 1.33.0 2023-07-01 [1] CRAN (R 4.3.0) + future.apply 1.11.0 2023-05-21 [1] CRAN (R 4.3.0) + generics 0.1.3 2022-07-05 [1] CRAN (R 4.3.0) + ggplot2 * 3.4.4 2023-10-12 [1] CRAN (R 4.3.0) + globals 0.16.2 2022-11-21 [1] CRAN (R 4.3.0) + glue 1.6.2 2022-02-24 [1] CRAN (R 4.3.0) + gower 1.0.1 2022-12-22 [1] CRAN (R 4.3.0) + GPfit 1.0-8 2019-02-08 [1] CRAN (R 4.3.0) + gtable 0.3.4 2023-08-21 [1] CRAN (R 4.3.0) + hardhat 1.3.0.9000 2023-11-13 [1] Github (tidymodels/hardhat@ac2dfd0) + igraph 1.5.1 2023-08-10 [1] CRAN (R 4.3.0) + infer * 1.0.5 2023-09-06 [1] CRAN (R 4.3.0) + inum 1.0-5 2023-03-09 [1] CRAN (R 4.3.0) + ipred 0.9-14 2023-03-09 [1] CRAN (R 4.3.0) + iterators 1.0.14 2022-02-05 [1] CRAN (R 4.3.0) + kernlab * 0.9-32 2023-01-31 [1] CRAN (R 4.3.0) + kknn * 1.3.1 2016-03-26 [1] CRAN (R 4.3.0) + lattice 0.21-9 2023-10-01 [1] CRAN (R 4.3.0) + lava 1.7.3 2023-11-04 [1] CRAN (R 4.3.0) + lhs 1.1.6 2022-12-17 [1] CRAN (R 4.3.0) + libcoin 1.0-10 2023-09-27 [1] CRAN (R 4.3.0) + lifecycle 1.0.4 2023-11-07 [1] CRAN (R 4.3.0) + listenv 0.9.0 2022-12-16 [1] CRAN (R 4.3.0) + lubridate 1.9.3 2023-09-27 [1] CRAN (R 4.3.0) + magrittr 2.0.3 2022-03-30 [1] CRAN (R 4.3.0) + MASS 7.3-60 2023-05-04 [2] CRAN (R 4.3.1) + Matrix 1.6-1.1 2023-09-18 [1] CRAN (R 4.3.0) + mboost * 2.9-8 2023-09-06 [1] CRAN (R 4.3.0) + modeldata * 1.2.0.9000 2023-11-13 [1] Github (tidymodels/modeldata@7ab5d8a) + modelenv 0.1.1 2023-03-08 [1] CRAN (R 4.3.0) + munsell 0.5.0 2018-06-12 [1] CRAN (R 4.3.0) + mvtnorm 1.2-3 2023-08-25 [1] CRAN (R 4.3.0) + nnet 7.3-19 2023-05-03 [2] CRAN (R 4.3.1) + nnls 1.5 2023-09-11 [1] CRAN (R 4.3.0) + parallelly 1.36.0 2023-05-26 [1] CRAN (R 4.3.0) + parsnip * 1.1.1.9002 2023-11-14 [1] Github (tidymodels/parsnip@8c08c65) + partykit 1.2-20 2023-04-14 [1] CRAN (R 4.3.0) + pillar 1.9.0 2023-03-22 [1] CRAN (R 4.3.0) + pkgconfig 2.0.3 2019-09-22 [1] CRAN (R 4.3.0) + pkgload 1.3.3 2023-09-22 [1] CRAN (R 4.3.0) + prodlim 2023.08.28 2023-08-28 [1] CRAN (R 4.3.0) + purrr * 1.0.2 2023-08-10 [1] CRAN (R 4.3.0) + quadprog 1.5-8 2019-11-20 [1] CRAN (R 4.3.0) + R6 2.5.1 2021-08-19 [1] CRAN (R 4.3.0) + Rcpp 1.0.11 2023-07-06 [1] CRAN (R 4.3.0) + recipes * 1.0.8.9000 2023-11-15 [1] Github (tidymodels/recipes@85e7fd2) + rlang 1.1.2 2023-11-04 [1] CRAN (R 4.3.0) + rpart 4.1.19 2022-10-21 [2] CRAN (R 4.3.1) + rsample * 1.2.0.9000 2023-11-13 [1] Github (tidymodels/rsample@f476210) + rstudioapi 0.15.0 2023-07-07 [1] CRAN (R 4.3.0) + scales * 1.3.0 2023-11-28 [1] CRAN (R 4.3.0) + sessioninfo * 1.2.2 2021-12-06 [1] CRAN (R 4.3.0) + stabs * 0.6-4 2021-01-29 [1] CRAN (R 4.3.0) + survival * 3.5-7 2023-08-14 [1] CRAN (R 4.3.0) + testthat * 3.2.0 2023-10-06 [1] CRAN (R 4.3.0) + tibble * 3.2.1 2023-03-20 [1] CRAN (R 4.3.0) + tidymodels * 1.1.1.9000 2023-11-13 [1] Github (tidymodels/tidymodels@4f77b25) + tidyr * 1.3.0 2023-01-24 [1] CRAN (R 4.3.0) + tidyselect 1.2.0 2022-10-10 [1] CRAN (R 4.3.0) + timechange 0.2.0 2023-01-11 [1] CRAN (R 4.3.0) + timeDate 4022.108 2023-01-07 [1] CRAN (R 4.3.0) + tune * 1.1.2.9002 2023-12-05 [1] local + utf8 1.2.4 2023-10-22 [1] CRAN (R 4.3.0) + vctrs 0.6.4 2023-10-12 [1] CRAN (R 4.3.0) + withr 2.5.2 2023-10-30 [1] CRAN (R 4.3.0) + workflows * 1.1.3.9000 2023-11-13 [1] Github (tidymodels/workflows@1413997) + workflowsets * 1.0.1 2023-04-06 [1] CRAN (R 4.3.0) + yardstick * 1.2.0.9001 2023-11-15 [1] Github (tidymodels/yardstick@ec51a38) + + [1] /Users/max/Library/R/x86_64/4.3/library + [2] /Library/Frameworks/R.framework/Versions/4.3-x86_64/Resources/library -[1] /Library/Frameworks/R.framework/Versions/4.0/Resources/library +────────────────────────────────────────────────────────────────────────────── > > if (!interactive()) { + q("no") + } > proc.time() - user system elapsed -1469.417 6.036 1479.245 + user system elapsed +508.954 3.959 513.165 diff --git a/man/choose_metric.Rd b/man/choose_metric.Rd index ff9a107c7..ce1790dfb 100644 --- a/man/choose_metric.Rd +++ b/man/choose_metric.Rd @@ -2,23 +2,33 @@ % Please edit documentation in R/metric-selection.R \name{choose_metric} \alias{choose_metric} +\alias{choose_eval_time} \alias{first_metric} \alias{first_eval_time} +\alias{.filter_perf_metrics} \title{Tools for selecting metrics and evaluation times} \usage{ choose_metric(x, metric, ..., call = rlang::caller_env()) +choose_eval_time(x, metric, eval_time = NULL, ..., call = rlang::caller_env()) + first_metric(mtr_set) first_eval_time(mtr_set, metric = NULL, eval_time = NULL) + +.filter_perf_metrics(x, metric, eval_time) } \arguments{ +\item{x}{An object with class \code{tune_results}.} + \item{metric}{A character value for which metric is being used.} -\item{mtr_set}{A \code{\link[yardstick:metric_set]{yardstick::metric_set()}}.} +\item{call}{The call to be displayed in warnings or errors.} \item{eval_time}{An optional vector of times to compute dynamic and/or integrated metrics.} + +\item{mtr_set}{A \code{\link[yardstick:metric_set]{yardstick::metric_set()}}.} } \description{ These are developer-facing functions used to compute and validate choices @@ -30,5 +40,13 @@ metrics. \code{\link[=select_best]{select_best()}} where a single valid metric is required to rank models. If no value is given by the user, the first metric value is used (with a warning). + +For evaluation times, one is only required when the metric type is dynamic +(e.g. \code{\link[yardstick:brier_survival]{yardstick::brier_survival()}} or \code{\link[yardstick:roc_auc_survival]{yardstick::roc_auc_survival()}}). For +these metrics, we require a single numeric value that was originally given +to the function used to produce \code{x} (such as \code{\link[=tune_grid]{tune_grid()}}). + +If a time is required and none is given, the first value in the vector +originally given in the \code{eval_time} argument is used (with a warning). } \keyword{internal} diff --git a/tests/testthat/_snaps/eval-time-single-selection.md b/tests/testthat/_snaps/eval-time-single-selection.md index 5d122bb3c..053732e10 100644 --- a/tests/testthat/_snaps/eval-time-single-selection.md +++ b/tests/testthat/_snaps/eval-time-single-selection.md @@ -4,7 +4,7 @@ stc_one <- first_eval_time(met_stc, eval_time = times_1) Condition Warning: - Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric. + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). --- @@ -12,7 +12,7 @@ stc_multi <- first_eval_time(met_stc, eval_time = times_2) Condition Warning: - Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric. + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). --- @@ -36,7 +36,7 @@ dyn_multi <- first_eval_time(met_dyn, eval_time = times_2) Condition Warning: - 2 evaluation times were available; the first (0.714) will be used. + 2 evaluation times were specified during tuning; the first (0.714) will be used. --- @@ -44,7 +44,7 @@ int_1 <- first_eval_time(met_int, eval_time = times_1) Condition Warning: - Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric. + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). --- @@ -52,7 +52,7 @@ int_multi <- first_eval_time(met_int, eval_time = times_2) Condition Warning: - Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric. + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). # selecting single eval time - mixed metric sets - static first @@ -60,7 +60,7 @@ stc_1 <- first_eval_time(met_mix_stc, eval_time = times_1) Condition Warning: - Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric. + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). --- @@ -68,7 +68,7 @@ stc_multi <- first_eval_time(met_mix_stc, eval_time = times_2) Condition Warning: - Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric. + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). --- @@ -76,7 +76,7 @@ stc_1 <- first_eval_time(met_mix_stc_all, eval_time = times_1) Condition Warning: - Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric. + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). --- @@ -84,7 +84,7 @@ stc_multi <- first_eval_time(met_mix_stc_all, eval_time = times_2) Condition Warning: - Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric. + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). # selecting single eval time - mixed metric sets - dynamic first @@ -100,7 +100,7 @@ dyn_multi <- first_eval_time(met_mix_dyn, eval_time = times_2) Condition Warning: - 2 evaluation times were available; the first (0.714) will be used. + 2 evaluation times were specified during tuning; the first (0.714) will be used. --- @@ -116,7 +116,7 @@ dyn_multi <- first_eval_time(met_mix_dyn_all, eval_time = times_2) Condition Warning: - 2 evaluation times were available; the first (0.714) will be used. + 2 evaluation times were specified during tuning; the first (0.714) will be used. # selecting single eval time - mixed metric sets - integrated first @@ -124,7 +124,7 @@ first_eval_time(met_mix_int, eval_time = times_1) Condition Warning: - Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric. + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). Output NULL @@ -134,7 +134,7 @@ int_multi <- first_eval_time(met_mix_int, eval_time = times_2) Condition Warning: - Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric. + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). --- @@ -142,7 +142,7 @@ first_eval_time(met_mix_int_all, eval_time = times_1) Condition Warning: - Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric. + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). Output NULL @@ -152,5 +152,42 @@ int_multi <- first_eval_time(met_mix_int_all, eval_time = times_2) Condition Warning: - Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric. + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). + +# selecting an evaluation time + + Code + choose_eval_time(surv_res, "brier_survival") + Condition + Warning: + 4 evaluation times were specified during tuning; the first (10) will be used. + Output + [1] 10 + +--- + + Code + choose_eval_time(surv_res, "concordance_survival") + Output + NULL + +--- + + Code + choose_eval_time(surv_res, "concordance_survival", eval_time = 10) + Condition + Warning: + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). + Output + NULL + +--- + + Code + choose_eval_time(ames_grid_search, "rmse", 1) + Condition + Warning: + Evaluation times are only required when the model mode is "censored regression" (and will be ignored). + Output + NULL diff --git a/tests/testthat/_snaps/select_best.md b/tests/testthat/_snaps/select_best.md index c6a8d068a..26947f383 100644 --- a/tests/testthat/_snaps/select_best.md +++ b/tests/testthat/_snaps/select_best.md @@ -213,3 +213,568 @@ Error in `select_by_pct_loss()`: ! Could not sort results by `weight_funk` and `desc(K)`. +# show_best with survival models + + Code + show_best(surv_res) + Condition + Warning in `show_best()`: + No value of `metric` was given; "brier_survival" will be used. + Warning: + 4 evaluation times were specified during tuning; the first (10) will be used. + Output + # A tibble: 5 x 8 + trees .metric .estimator .eval_time mean n std_err .config + + 1 100 brier_survival standard 10 0.118 10 0.0177 Preprocessor1_~ + 2 20 brier_survival standard 10 0.136 10 0.0153 Preprocessor1_~ + 3 10 brier_survival standard 10 0.155 10 0.0175 Preprocessor1_~ + 4 5 brier_survival standard 10 0.172 10 0.0198 Preprocessor1_~ + 5 1 brier_survival standard 10 0.194 10 0.0221 Preprocessor1_~ + +--- + + Code + show_best(surv_res, metric = "concordance_survival") + Output + # A tibble: 5 x 8 + trees .metric .estimator .eval_time mean n std_err .config + + 1 20 concordance_survival standard NA 0.677 10 0.0354 Preproce~ + 2 100 concordance_survival standard NA 0.670 10 0.0329 Preproce~ + 3 10 concordance_survival standard NA 0.668 10 0.0376 Preproce~ + 4 5 concordance_survival standard NA 0.663 10 0.0363 Preproce~ + 5 1 concordance_survival standard NA 0.626 10 0.0326 Preproce~ + +--- + + Code + show_best(surv_res, metric = "brier_survival_integrated") + Output + # A tibble: 5 x 8 + trees .metric .estimator .eval_time mean n std_err .config + + 1 100 brier_survival_integr~ standard NA 0.117 10 0.00699 Prepro~ + 2 20 brier_survival_integr~ standard NA 0.127 10 0.00981 Prepro~ + 3 10 brier_survival_integr~ standard NA 0.141 10 0.0134 Prepro~ + 4 5 brier_survival_integr~ standard NA 0.151 10 0.0159 Prepro~ + 5 1 brier_survival_integr~ standard NA 0.164 10 0.0182 Prepro~ + +--- + + Code + show_best(surv_res, metric = "brier_survival") + Condition + Warning: + 4 evaluation times were specified during tuning; the first (10) will be used. + Output + # A tibble: 5 x 8 + trees .metric .estimator .eval_time mean n std_err .config + + 1 100 brier_survival standard 10 0.118 10 0.0177 Preprocessor1_~ + 2 20 brier_survival standard 10 0.136 10 0.0153 Preprocessor1_~ + 3 10 brier_survival standard 10 0.155 10 0.0175 Preprocessor1_~ + 4 5 brier_survival standard 10 0.172 10 0.0198 Preprocessor1_~ + 5 1 brier_survival standard 10 0.194 10 0.0221 Preprocessor1_~ + +--- + + Code + show_best(surv_res, metric = c("brier_survival", "roc_auc_survival")) + Condition + Warning in `show_best()`: + 2 metrics were given; "brier_survival" will be used. + Warning: + 4 evaluation times were specified during tuning; the first (10) will be used. + Output + # A tibble: 5 x 8 + trees .metric .estimator .eval_time mean n std_err .config + + 1 100 brier_survival standard 10 0.118 10 0.0177 Preprocessor1_~ + 2 20 brier_survival standard 10 0.136 10 0.0153 Preprocessor1_~ + 3 10 brier_survival standard 10 0.155 10 0.0175 Preprocessor1_~ + 4 5 brier_survival standard 10 0.172 10 0.0198 Preprocessor1_~ + 5 1 brier_survival standard 10 0.194 10 0.0221 Preprocessor1_~ + +--- + + Code + show_best(surv_res, metric = "brier_survival", eval_time = 1) + Output + # A tibble: 5 x 8 + trees .metric .estimator .eval_time mean n std_err .config + + 1 20 brier_survival standard 1 0.0381 10 0.0149 Preprocessor1~ + 2 10 brier_survival standard 1 0.0386 10 0.0151 Preprocessor1~ + 3 100 brier_survival standard 1 0.0386 10 0.0147 Preprocessor1~ + 4 5 brier_survival standard 1 0.0389 10 0.0152 Preprocessor1~ + 5 1 brier_survival standard 1 0.0392 10 0.0153 Preprocessor1~ + +--- + + Code + show_best(surv_res, metric = "concordance_survival", eval_time = 1) + Condition + Warning: + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). + Output + # A tibble: 5 x 8 + trees .metric .estimator .eval_time mean n std_err .config + + 1 20 concordance_survival standard NA 0.677 10 0.0354 Preproce~ + 2 100 concordance_survival standard NA 0.670 10 0.0329 Preproce~ + 3 10 concordance_survival standard NA 0.668 10 0.0376 Preproce~ + 4 5 concordance_survival standard NA 0.663 10 0.0363 Preproce~ + 5 1 concordance_survival standard NA 0.626 10 0.0326 Preproce~ + +--- + + Code + show_best(surv_res, metric = "concordance_survival", eval_time = 1.1) + Condition + Warning: + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). + Output + # A tibble: 5 x 8 + trees .metric .estimator .eval_time mean n std_err .config + + 1 20 concordance_survival standard NA 0.677 10 0.0354 Preproce~ + 2 100 concordance_survival standard NA 0.670 10 0.0329 Preproce~ + 3 10 concordance_survival standard NA 0.668 10 0.0376 Preproce~ + 4 5 concordance_survival standard NA 0.663 10 0.0363 Preproce~ + 5 1 concordance_survival standard NA 0.626 10 0.0326 Preproce~ + +--- + + Code + show_best(surv_res, metric = "brier_survival", eval_time = 1.1) + Condition + Error in `show_best()`: + ! Evaluation time 1.1 is not in the results. + +--- + + Code + show_best(surv_res, metric = "brier_survival", eval_time = 1:2) + Condition + Warning: + 2 evaluation times were specified during tuning; the first (1) will be used. + Output + # A tibble: 5 x 8 + trees .metric .estimator .eval_time mean n std_err .config + + 1 20 brier_survival standard 1 0.0381 10 0.0149 Preprocessor1~ + 2 10 brier_survival standard 1 0.0386 10 0.0151 Preprocessor1~ + 3 100 brier_survival standard 1 0.0386 10 0.0147 Preprocessor1~ + 4 5 brier_survival standard 1 0.0389 10 0.0152 Preprocessor1~ + 5 1 brier_survival standard 1 0.0392 10 0.0153 Preprocessor1~ + +--- + + Code + show_best(surv_res, metric = "brier_survival", eval_time = 3:4) + Condition + Warning: + 2 evaluation times were specified during tuning; the first (3) will be used. + Error in `show_best()`: + ! Evaluation time 3 is not in the results. + +# select_best with survival models + + Code + select_best(surv_res) + Condition + Warning in `select_best()`: + No value of `metric` was given; "brier_survival" will be used. + Warning: + 4 evaluation times were specified during tuning; the first (10) will be used. + Output + # A tibble: 1 x 2 + trees .config + + 1 100 Preprocessor1_Model5 + +--- + + Code + select_best(surv_res, metric = "concordance_survival") + Output + # A tibble: 1 x 2 + trees .config + + 1 20 Preprocessor1_Model4 + +--- + + Code + select_best(surv_res, metric = "brier_survival_integrated") + Output + # A tibble: 1 x 2 + trees .config + + 1 100 Preprocessor1_Model5 + +--- + + Code + select_best(surv_res, metric = "brier_survival") + Condition + Warning: + 4 evaluation times were specified during tuning; the first (10) will be used. + Output + # A tibble: 1 x 2 + trees .config + + 1 100 Preprocessor1_Model5 + +--- + + Code + select_best(surv_res, metric = c("brier_survival", "roc_auc_survival")) + Condition + Warning in `select_best()`: + 2 metrics were given; "brier_survival" will be used. + Warning: + 4 evaluation times were specified during tuning; the first (10) will be used. + Output + # A tibble: 1 x 2 + trees .config + + 1 100 Preprocessor1_Model5 + +--- + + Code + select_best(surv_res, metric = "brier_survival", eval_time = 1) + Output + # A tibble: 1 x 2 + trees .config + + 1 20 Preprocessor1_Model4 + +--- + + Code + select_best(surv_res, metric = "concordance_survival", eval_time = 1) + Condition + Warning: + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). + Output + # A tibble: 1 x 2 + trees .config + + 1 20 Preprocessor1_Model4 + +--- + + Code + select_best(surv_res, metric = "concordance_survival", eval_time = 1.1) + Condition + Warning: + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). + Output + # A tibble: 1 x 2 + trees .config + + 1 20 Preprocessor1_Model4 + +--- + + Code + select_best(surv_res, metric = "brier_survival", eval_time = 1.1) + Condition + Error in `show_best()`: + ! Evaluation time 1.1 is not in the results. + +--- + + Code + select_best(surv_res, metric = "brier_survival", eval_time = 1:2) + Condition + Warning: + 2 evaluation times were specified during tuning; the first (1) will be used. + Output + # A tibble: 1 x 2 + trees .config + + 1 20 Preprocessor1_Model4 + +--- + + Code + select_best(surv_res, metric = "brier_survival", eval_time = 3:4) + Condition + Warning: + 2 evaluation times were specified during tuning; the first (3) will be used. + Error in `show_best()`: + ! Evaluation time 3 is not in the results. + +# select_by_one_std_err with survival models + + Code + select_by_one_std_err(surv_res, trees) + Condition + Warning in `select_by_one_std_err()`: + No value of `metric` was given; "brier_survival" will be used. + Warning: + 4 evaluation times were specified during tuning; the first (10) will be used. + Output + # A tibble: 1 x 2 + trees .config + + 1 20 Preprocessor1_Model4 + +--- + + Code + select_by_one_std_err(surv_res, metric = "concordance_survival", trees) + Output + # A tibble: 1 x 2 + trees .config + + 1 5 Preprocessor1_Model2 + +--- + + Code + select_by_one_std_err(surv_res, metric = "brier_survival_integrated", trees) + Output + # A tibble: 1 x 2 + trees .config + + 1 100 Preprocessor1_Model5 + +--- + + Code + select_by_one_std_err(surv_res, metric = "brier_survival", trees) + Condition + Warning: + 4 evaluation times were specified during tuning; the first (10) will be used. + Output + # A tibble: 1 x 2 + trees .config + + 1 20 Preprocessor1_Model4 + +--- + + Code + select_by_one_std_err(surv_res, metric = c("brier_survival", "roc_auc_survival"), + trees) + Condition + Warning in `select_by_one_std_err()`: + 2 metrics were given; "brier_survival" will be used. + Warning: + 4 evaluation times were specified during tuning; the first (10) will be used. + Output + # A tibble: 1 x 2 + trees .config + + 1 20 Preprocessor1_Model4 + +--- + + Code + select_by_one_std_err(surv_res, metric = "brier_survival", eval_time = 1, trees) + Output + # A tibble: 1 x 2 + trees .config + + 1 1 Preprocessor1_Model1 + +--- + + Code + select_by_one_std_err(surv_res, metric = "concordance_survival", eval_time = 1, + trees) + Condition + Warning: + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). + Output + # A tibble: 1 x 2 + trees .config + + 1 5 Preprocessor1_Model2 + +--- + + Code + select_by_one_std_err(surv_res, metric = "concordance_survival", eval_time = 1.1, + trees) + Condition + Warning: + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). + Output + # A tibble: 1 x 2 + trees .config + + 1 5 Preprocessor1_Model2 + +--- + + Code + select_by_one_std_err(surv_res, metric = "brier_survival", eval_time = 1.1, + trees) + Condition + Error in `select_by_one_std_err()`: + ! Evaluation time 1.1 is not in the results. + +--- + + Code + select_by_one_std_err(surv_res, metric = "brier_survival", eval_time = 1:2, + trees) + Condition + Warning: + 2 evaluation times were specified during tuning; the first (1) will be used. + Output + # A tibble: 1 x 2 + trees .config + + 1 1 Preprocessor1_Model1 + +--- + + Code + select_by_one_std_err(surv_res, metric = "brier_survival", eval_time = 3:4, + trees) + Condition + Warning: + 2 evaluation times were specified during tuning; the first (3) will be used. + Error in `select_by_one_std_err()`: + ! Evaluation time 3 is not in the results. + +# select_by_pct_loss with survival models + + Code + select_by_pct_loss(surv_res, trees) + Condition + Warning in `select_by_pct_loss()`: + No value of `metric` was given; "brier_survival" will be used. + Warning: + 4 evaluation times were specified during tuning; the first (10) will be used. + Output + # A tibble: 1 x 2 + trees .config + + 1 100 Preprocessor1_Model5 + +--- + + Code + select_by_pct_loss(surv_res, metric = "concordance_survival", trees) + Output + # A tibble: 1 x 2 + trees .config + + 1 10 Preprocessor1_Model3 + +--- + + Code + select_by_pct_loss(surv_res, metric = "brier_survival_integrated", trees) + Output + # A tibble: 1 x 2 + trees .config + + 1 100 Preprocessor1_Model5 + +--- + + Code + select_by_pct_loss(surv_res, metric = "brier_survival", trees) + Condition + Warning: + 4 evaluation times were specified during tuning; the first (10) will be used. + Output + # A tibble: 1 x 2 + trees .config + + 1 100 Preprocessor1_Model5 + +--- + + Code + select_by_pct_loss(surv_res, metric = c("brier_survival", "roc_auc_survival"), + trees) + Condition + Warning in `select_by_pct_loss()`: + 2 metrics were given; "brier_survival" will be used. + Warning: + 4 evaluation times were specified during tuning; the first (10) will be used. + Output + # A tibble: 1 x 2 + trees .config + + 1 100 Preprocessor1_Model5 + +--- + + Code + select_by_pct_loss(surv_res, metric = "brier_survival", eval_time = 1, trees) + Output + # A tibble: 1 x 2 + trees .config + + 1 5 Preprocessor1_Model2 + +--- + + Code + select_by_pct_loss(surv_res, metric = "concordance_survival", eval_time = 1, + trees) + Condition + Warning: + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). + Output + # A tibble: 1 x 2 + trees .config + + 1 10 Preprocessor1_Model3 + +--- + + Code + select_by_pct_loss(surv_res, metric = "concordance_survival", eval_time = 1.1, + trees) + Condition + Warning: + Evaluation times are only required when dynmanic or integrated metrics are selected as the primary metric (and will be ignored). + Output + # A tibble: 1 x 2 + trees .config + + 1 10 Preprocessor1_Model3 + +--- + + Code + select_by_pct_loss(surv_res, metric = "brier_survival", eval_time = 1.1, trees) + Condition + Error in `select_by_pct_loss()`: + ! Evaluation time 1.1 is not in the results. + +--- + + Code + select_by_pct_loss(surv_res, metric = "brier_survival", eval_time = 1:2, trees) + Condition + Warning: + 2 evaluation times were specified during tuning; the first (1) will be used. + Output + # A tibble: 1 x 2 + trees .config + + 1 5 Preprocessor1_Model2 + +--- + + Code + select_by_pct_loss(surv_res, metric = "brier_survival", eval_time = 3:4, trees) + Condition + Warning: + 2 evaluation times were specified during tuning; the first (3) will be used. + Error in `select_by_pct_loss()`: + ! Evaluation time 3 is not in the results. + diff --git a/tests/testthat/data/knn_gp.rds b/tests/testthat/data/knn_gp.rds index d67d715e6..a35d8419c 100644 Binary files a/tests/testthat/data/knn_gp.rds and b/tests/testthat/data/knn_gp.rds differ diff --git a/tests/testthat/data/knn_grid.rds b/tests/testthat/data/knn_grid.rds index 7758845ce..f7c4f2598 100644 Binary files a/tests/testthat/data/knn_grid.rds and b/tests/testthat/data/knn_grid.rds differ diff --git a/tests/testthat/data/knn_results.rds b/tests/testthat/data/knn_results.rds index 03636035a..ed196068c 100644 Binary files a/tests/testthat/data/knn_results.rds and b/tests/testthat/data/knn_results.rds differ diff --git a/tests/testthat/data/knn_set.rds b/tests/testthat/data/knn_set.rds index 88ab01762..34ee8d577 100644 Binary files a/tests/testthat/data/knn_set.rds and b/tests/testthat/data/knn_set.rds differ diff --git a/tests/testthat/data/lm_bayes.rds b/tests/testthat/data/lm_bayes.rds index 1284e0304..17949e120 100644 Binary files a/tests/testthat/data/lm_bayes.rds and b/tests/testthat/data/lm_bayes.rds differ diff --git a/tests/testthat/data/lm_resamples.rds b/tests/testthat/data/lm_resamples.rds index 4dde5201f..790fc384b 100644 Binary files a/tests/testthat/data/lm_resamples.rds and b/tests/testthat/data/lm_resamples.rds differ diff --git a/tests/testthat/data/rcv_results.rds b/tests/testthat/data/rcv_results.rds index 9d04601ad..8d3bcccb0 100644 Binary files a/tests/testthat/data/rcv_results.rds and b/tests/testthat/data/rcv_results.rds differ diff --git a/tests/testthat/data/surv_boost_tree_res.rds b/tests/testthat/data/surv_boost_tree_res.rds new file mode 100644 index 000000000..84e432d87 Binary files /dev/null and b/tests/testthat/data/surv_boost_tree_res.rds differ diff --git a/tests/testthat/data/svm_reg_results.rds b/tests/testthat/data/svm_reg_results.rds index 8242f9477..ac4d522ea 100644 Binary files a/tests/testthat/data/svm_reg_results.rds and b/tests/testthat/data/svm_reg_results.rds differ diff --git a/tests/testthat/data/svm_results.rds b/tests/testthat/data/svm_results.rds index a0df56695..10861218f 100644 Binary files a/tests/testthat/data/svm_results.rds and b/tests/testthat/data/svm_results.rds differ diff --git a/tests/testthat/data/test_objects.RData b/tests/testthat/data/test_objects.RData index f5dcfdce5..5f035cb66 100644 Binary files a/tests/testthat/data/test_objects.RData and b/tests/testthat/data/test_objects.RData differ diff --git a/tests/testthat/test-eval-time-single-selection.R b/tests/testthat/test-eval-time-single-selection.R index e2466e6df..181709da7 100644 --- a/tests/testthat/test-eval-time-single-selection.R +++ b/tests/testthat/test-eval-time-single-selection.R @@ -202,3 +202,22 @@ test_that("selecting single eval time - mixed metric sets - integrated first", { expect_null(int_multi) }) + +test_that("selecting an evaluation time", { + # much of this is indirectly tested in show/select best + + surv_res <- readRDS(test_path("data", "surv_boost_tree_res.rds")) + + expect_snapshot( + choose_eval_time(surv_res, "brier_survival") + ) + expect_snapshot( + choose_eval_time(surv_res, "concordance_survival") + ) + expect_snapshot( + choose_eval_time(surv_res, "concordance_survival", eval_time = 10) + ) + + data("example_ames_knn") + expect_snapshot(choose_eval_time(ames_grid_search, "rmse", 1)) +}) diff --git a/tests/testthat/test-select_best.R b/tests/testthat/test-select_best.R index 5a52a408b..7a93bb6b3 100644 --- a/tests/testthat/test-select_best.R +++ b/tests/testthat/test-select_best.R @@ -338,3 +338,184 @@ test_that("select_by_* can handle metrics with direction == 'zero'", { pull() ) }) + +test_that("show_best with survival models", { + surv_res <- readRDS(test_path("data", "surv_boost_tree_res.rds")) + + expect_snapshot( + show_best(surv_res) + ) + expect_snapshot( + show_best(surv_res, metric = "concordance_survival") + ) + expect_snapshot( + show_best(surv_res, metric = "brier_survival_integrated") + ) + expect_snapshot( + show_best(surv_res, metric = "brier_survival") + ) + expect_snapshot( + show_best(surv_res, metric = c("brier_survival", "roc_auc_survival")) + ) + expect_snapshot( + show_best(surv_res, metric = "brier_survival", eval_time = 1) + ) + expect_snapshot( + show_best(surv_res, metric = "concordance_survival", eval_time = 1) + ) + expect_snapshot( + show_best(surv_res, metric = "concordance_survival", eval_time = 1.1) + ) + expect_snapshot( + show_best(surv_res, metric = "brier_survival", eval_time = 1.1), + error = TRUE + ) + expect_snapshot( + show_best(surv_res, metric = "brier_survival", eval_time = 1:2) + ) + expect_snapshot( + show_best(surv_res, metric = "brier_survival", eval_time = 3:4), + error = TRUE + ) + +}) + +test_that("select_best with survival models", { + surv_res <- readRDS(test_path("data", "surv_boost_tree_res.rds")) + + expect_snapshot( + select_best(surv_res) + ) + expect_snapshot( + select_best(surv_res, metric = "concordance_survival") + ) + expect_snapshot( + select_best(surv_res, metric = "brier_survival_integrated") + ) + expect_snapshot( + select_best(surv_res, metric = "brier_survival") + ) + expect_snapshot( + select_best(surv_res, metric = c("brier_survival", "roc_auc_survival")) + ) + expect_snapshot( + select_best(surv_res, metric = "brier_survival", eval_time = 1) + ) + expect_snapshot( + select_best(surv_res, metric = "concordance_survival", eval_time = 1) + ) + expect_snapshot( + select_best(surv_res, metric = "concordance_survival", eval_time = 1.1) + ) + expect_snapshot( + select_best(surv_res, metric = "brier_survival", eval_time = 1.1), + error = TRUE + ) + expect_snapshot( + select_best(surv_res, metric = "brier_survival", eval_time = 1:2) + ) + expect_snapshot( + select_best(surv_res, metric = "brier_survival", eval_time = 3:4), + error = TRUE + ) + +}) + +test_that("select_by_one_std_err with survival models", { + surv_res <- readRDS(test_path("data", "surv_boost_tree_res.rds")) + + expect_snapshot( + select_by_one_std_err(surv_res, trees) + ) + expect_snapshot( + select_by_one_std_err(surv_res, metric = "concordance_survival", trees) + ) + expect_snapshot( + select_by_one_std_err(surv_res, metric = "brier_survival_integrated", trees) + ) + expect_snapshot( + select_by_one_std_err(surv_res, metric = "brier_survival", trees) + ) + expect_snapshot( + select_by_one_std_err(surv_res, metric = c("brier_survival", "roc_auc_survival"), + trees) + ) + expect_snapshot( + select_by_one_std_err(surv_res, metric = "brier_survival", + eval_time = 1, trees) + ) + expect_snapshot( + select_by_one_std_err(surv_res, metric = "concordance_survival", + eval_time = 1, trees) + ) + expect_snapshot( + select_by_one_std_err(surv_res, metric = "concordance_survival", + eval_time = 1.1, trees) + ) + expect_snapshot( + select_by_one_std_err(surv_res, metric = "brier_survival", + eval_time = 1.1, trees), + error = TRUE + ) + expect_snapshot( + select_by_one_std_err(surv_res, metric = "brier_survival", + eval_time = 1:2, trees) + ) + expect_snapshot( + select_by_one_std_err(surv_res, metric = "brier_survival", + eval_time = 3:4, trees), + error = TRUE + ) + +}) + +test_that("select_by_pct_loss with survival models", { + surv_res <- readRDS(test_path("data", "surv_boost_tree_res.rds")) + + expect_snapshot( + select_by_pct_loss(surv_res, trees) + ) + expect_snapshot( + select_by_pct_loss(surv_res, metric = "concordance_survival", trees) + ) + expect_snapshot( + select_by_pct_loss(surv_res, metric = "brier_survival_integrated", trees) + ) + expect_snapshot( + select_by_pct_loss(surv_res, metric = "brier_survival", trees) + ) + expect_snapshot( + select_by_pct_loss(surv_res, metric = c("brier_survival", "roc_auc_survival"), + trees) + ) + expect_snapshot( + select_by_pct_loss(surv_res, metric = "brier_survival", + eval_time = 1, trees) + ) + expect_snapshot( + select_by_pct_loss(surv_res, metric = "concordance_survival", + eval_time = 1, trees) + ) + expect_snapshot( + select_by_pct_loss(surv_res, metric = "concordance_survival", + eval_time = 1.1, trees) + ) + expect_snapshot( + select_by_pct_loss(surv_res, metric = "brier_survival", + eval_time = 1.1, trees), + error = TRUE + ) + expect_snapshot( + select_by_pct_loss(surv_res, metric = "brier_survival", + eval_time = 1:2, trees) + ) + expect_snapshot( + select_by_pct_loss(surv_res, metric = "brier_survival", + eval_time = 3:4, trees), + error = TRUE + ) + +}) + + +