-
Notifications
You must be signed in to change notification settings - Fork 1
/
benchmarks.Rmd
99 lines (81 loc) · 2.44 KB
/
benchmarks.Rmd
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
---
title: "Benchmarks"
output:
bookdown::html_document2:
fig.width: 18
fig.height: 8
---
```{r benchmarks_setup, message = FALSE}
library("targets")
library("data.table")
library("mlr3")
library("mlr3misc")
library("mlr3proba")
library("mlr3viz")
library("viridis")
library("ggplot2")
tar_load(bmrk_results)
tar_load(bmrk_aggr)
agg <- bmrk_results$aggregate(msr("surv.cindex", id = "harrell"))
agg[, `:=`
(nr = NULL, resample_result = NULL, resampling_id = NULL, iters = NULL)
]
setorder(agg, task_id, -harrell)
p.value <- 0.05
```
# Overview
## Table
```{r benchmarks_table, results = "asis"}
lapply(split(agg, agg$task_id), knitr::kable, digits = 4)
```
## Boxplots
```{r benchmarks_plot_c_harrell, fig.width = 18, fig.height = 12}
m <- agg[, max(harrell), by = task_id]
autoplot(bmrk_results) +
geom_boxplot(aes(fill = learner_id)) +
geom_jitter(position = position_jitter(0.2)) +
scale_fill_viridis(discrete = TRUE) +
geom_hline(
aes(yintercept = V1), linetype = "dashed", color = "red", data = m
)
```
```{r benchmarks_plot_c_uno, fig.width = 18, fig.height = 12}
autoplot(
bmrk_results,
measure = msr("surv.cindex", id = "uno", weight_meth = "G2")) +
geom_boxplot(aes(fill = learner_id)) +
geom_jitter(position = position_jitter(0.2)) +
scale_fill_viridis(discrete = TRUE)
```
# Aggregated performance
Aggregated performance across all `r nrow(bmrk_results$tasks)` datasets.
```{r benchmarks_aggr_plot_c_harrell}
autoplot(bmrk_aggr, type = "box", meas = "harrell") +
geom_boxplot(aes(fill = learner_id)) +
geom_jitter(position = position_jitter(0.2)) +
scale_fill_viridis(discrete = TRUE)
```
```{r benchmarks_aggr_plot_c_uno}
autoplot(bmrk_aggr, type = "box", meas = "uno") +
geom_boxplot(aes(fill = learner_id)) +
geom_jitter(position = position_jitter(0.2)) +
scale_fill_viridis(discrete = TRUE)
```
# Pairwise comparison
*p*-value is set to `r p.value`
(otherwise an error is thrown because of insignificant results).
```{r benchmarks_aggr_pairwise_harrell_c}
autoplot(
bmrk_aggr, type = "fn", meas = "harrell", p.value = p.value,
col = palette.colors(2L)[2L]
)
```
# Critical difference
*p*-value is set to `r p.value`
(otherwise an error is thrown because of insignificant results).
```{r benchmarks_aggr_critdiff_c_harrell}
autoplot(
bmrk_aggr, type = "cd", meas = "harrell", style = 2, p.value = p.value,
minimize = FALSE
) + scale_color_viridis(discrete = TRUE)
```