random forest revision

slides/forests/chapter-order.tex

@@ -10,17 +10,17 @@
 \subsection{Bagging Ensembles}
 \includepdf[pages=-]{../../slides-pdf/slides-forests-bagging.pdf}
 
-\subsection{Introduction}
-\includepdf[pages=-]{../../slides-pdf/slides-forests-intro.pdf}
+\subsection{Basics}
+\includepdf[pages=-]{../../slides-pdf/slides-forests-basics.pdf}
 
-\subsection{Benchmarking Trees, Forests, and Bagging K-NN}
-\includepdf[pages=-]{../../slides-pdf/slides-forests-benchmark.pdf}
+\subsection{Out-of-Bag Error Estimate}
+\includepdf[pages=-]{../../slides-pdf/slides-forests-oob.pdf}
 
 \subsection{Feature Importance}
 \includepdf[pages=-]{../../slides-pdf/slides-forests-featureimportance.pdf}
 
 \subsection{Proximities}
 \includepdf[pages=-]{../../slides-pdf/slides-forests-proximities.pdf}
 
-\subsection{Discussion}
-\includepdf[pages=-]{../../slides-pdf/slides-forests-discussion.pdf}
+% \subsection{Bagging: Deep Dive}
+% \includepdf[pages=-]{../../slides-pdf/slides-forests-bagging-deepdive.pdf}

slides/forests/references.bib

@@ -0,0 +1,52 @@
+@article{STROBL2007,
+  author = {Strobl, Carolin and Boulesteix, Anne-Laure and Zeileis, Achim and Hothorn, Torsten},
+  journal = {BMC Bioinformatics},
+  number = 25,
+  title = {Bias in Random Forest Variable Importance Measures: Illustrations, Sources and a Solution},
+  volume = 8,
+  year = 2007,
+  url = {https://bmcbioinformatics.biomedcentral.com/articles/10.1186/1471-2105-8-25}
+}
+
+
+@book{HASTIE2001,
+  author = {Hastie, Trevor and Tibshirani, Robert and Friedman, Jerome},
+  edition = 2,
+  publisher = {Springer},
+  title = {The elements of statistical learning: data mining, inference and prediction},
+  url = {http://www-stat.stanford.edu/~tibs/ElemStatLearn/},
+  year = 2009
+}
+
+@article{BREIMAN2001,
+  author = {Breiman, Leo},
+  journal = {Machine Learning},
+  number = 1,
+  pages = {5-32},
+  publisher = {Kluwer Academic Publishers},
+  title = {Random Forests},
+  url = {http://dx.doi.org/10.1023/A%3A1010933404324},
+  volume = 45,
+  year = 2001
+}
+
+@misc{LOUPPE2015,
+      title={Understanding Random Forests: From Theory to Practice},
+      author={Gilles Louppe},
+      year={2015},
+      eprint={1407.7502},
+      archivePrefix={arXiv},
+      primaryClass={stat.ML},
+    url = {https://arxiv.org/abs/1407.7502}
+}
+
+@Article{PROBST2018,
+    title = {To Tune or Not to Tune the Number of Trees in Random Forest},
+    author = {Philipp Probst and Anne-Laure Boulesteix},
+    journal = {Journal of Machine Learning Research},
+    year = {2018},
+    volume = {18},
+    number = {181},
+    pages = {1-18},
+    url = {http://jmlr.org/papers/v18/17-269.html},
+}

slides/forests/rsrc/fig-bagging-bench.R

@@ -0,0 +1,62 @@
+# goal here is to visualize the need for unstable learners in bagging
+# by using a benchmark_grid from mlr3
+# and later show how RFs further improve accuracy!
+
+library(mlr3)
+library(mlr3learners)
+library(mlr3pipelines)
+library(mlr3viz)
+library(ggplot2)
+library(data.table)
+
+set.seed(123)
+
+# bagging via a pipeline (taken from mlr3 book)
+create_bagging_pipeline <- function(base_learner) {
+  gr_single_pred = po("subsample", frac = 1, replace=TRUE) %>>% lrn(base_learner) # equals bootstrapping (with replacement, frac = 1)
+  gr_pred_set = ppl("greplicate", graph = gr_single_pred, n = 100)
+  gr_bagging = gr_pred_set %>>% po("classifavg", innum = 100)
+  as_learner(gr_bagging)
+}
+
+# setup learners
+glrn_bagging_log = create_bagging_pipeline("classif.log_reg")
+glrn_bagging_log$id = "bagging_logistic"
+
+glrn_bagging_rpart = create_bagging_pipeline("classif.rpart")
+glrn_bagging_rpart$id = "bagging_tree"
+
+glrn_bagging_kknn = create_bagging_pipeline("classif.kknn")
+glrn_bagging_kknn$id = "bagging_kknn"
+
+lrn_log_reg = lrn("classif.log_reg")
+lrn_rpart = lrn("classif.rpart")
+
+lrn_ranger = lrn("classif.ranger", num.trees = 100)
+
+lrn_kknn = lrn("classif.kknn")
+lrn_kknn$param_set$values$k = 7
+
+# benchmark_grid expects a list:
+learners = list(glrn_bagging_log, lrn_log_reg, glrn_bagging_rpart, lrn_rpart, glrn_bagging_kknn, lrn_kknn, lrn_ranger)
+
+# tasks to be included in the benchmark
+tasks = lapply(c("spam"), tsk)
+
+# run the benchmark!
+bmr = benchmark(benchmark_grid(tasks, learners, rsmp("cv", folds = 10)))
+
+# visualization
+a <- autoplot(bmr, type = "boxplot") +
+  ylab("CE for 10-fold CV") +
+  xlab("Learners") +
+  scale_x_discrete(labels = c("LR bagged", "LR", "CART bagged", "CART", "7-nn bagged", "7-nn", "RF")) +
+  theme_minimal() +
+  theme(
+    axis.title = element_text(size = 22, face = "bold"),
+    axis.text = element_text(size = 20, face = "bold"),
+    legend.title = element_text(size = 22, face = "bold"),
+    legend.text = element_text(size = 20, face = "bold"),
+  )
+ggsave("../figure/bagging-bench_RF.png", plot = a, width = 20, height = 8, dpi = 300)
+# the bagging-bench figure is just a cropped version of this (sorry, had problems with autoplot)

slides/forests/rsrc/fig-bagging-mean.R

@@ -0,0 +1,84 @@
+# we want to visualize how bagging works, e.g.
+# that the predicitions of the base learners are averaged
+# -> creates 100 trees on bootstrapped toy data; visualizes them with their respective average
+# additionally, plots ntrees vs MSE to show benefits of large examples
+
+library(ggplot2)
+library(rpart)
+library(gridExtra)
+
+set.seed(700)
+
+# Generate toy data
+n <- 700
+x <- runif(n, 0, 10)
+y <- sin(x) + rnorm(n, sd=0.6)
+data <- data.frame(x=x, y=y)
+
+# Fit data on bootstrapped toy data
+num_trees <- 100
+trees <- lapply(1:num_trees, function(i) {
+  samp <- sample(n, replace=TRUE)
+  rpart(y ~ x, data=data[samp,], method="anova")
+})
+x_seq <- seq(0, 10, length.out=250)
+preds <- sapply(trees, function(tree) predict(tree, newdata=data.frame(x=x_seq)))
+
+# Mean acquired via bagging
+mean_preds <- rowMeans(preds)
+
+# Combined into a dataframe for plotting
+plot_df <- data.frame(x = rep(x_seq, each = num_trees + 1), 
+                      y = c(t(preds), mean_preds),
+                      Model = factor(c(rep("individual trees", num_trees), "bagged mean")))
+
+# Visualization of toy data
+p1 <- ggplot(data, aes(x=x, y=y)) +
+  geom_point(alpha=0.5) +
+  stat_function(fun = sin, color = "red") +
+  theme_minimal(base_size = 33)
+
+# visualization of tree's predictions and mean (bagged)
+p2 <- ggplot(plot_df, aes(x=x, y=y, color=Model)) +
+  geom_line(alpha=0.3) +
+  geom_line(data = subset(plot_df, Model == "bagged mean"), size=1.0) +
+  theme_minimal(base_size = 33)
+
+# function to calculate MSE for different numbers of trees
+bagging_rpart <- function(data, num_trees, sample_size) {
+  predictions <- matrix(NA, nrow = nrow(data), ncol = num_trees)
+
+  for (i in 1:num_trees) {
+    sample_indices <- sample(nrow(data), size = sample_size, replace = TRUE)
+    sample_data <- data[sample_indices, ]
+
+    model <- rpart(y ~ x, data = sample_data)
+
+    predictions[, i] <- predict(model, data)
+  }
+
+  mean_predictions <- rowMeans(predictions, na.rm = TRUE)
+  error <- mean((mean_predictions - data$y)^2)
+
+  return(error)
+}
+
+# calculate MSE for different numbers of trees using the toy data
+results <- data.frame(Number_of_Trees = integer(), MSE = numeric())
+tree_counts <- seq(1, num_trees, by = 2)
+
+for (trees in tree_counts) {
+  mse <- bagging_rpart(data, num_trees = trees, sample_size = 700)
+  results <- rbind(results, data.frame(Number_of_Trees = trees, MSE = mse))
+}
+
+# plot MSE vs. number of trees
+p3 <- ggplot(results, aes(x = Number_of_Trees, y = MSE)) +
+  geom_line(color = "blue", size = 1.5) +
+  labs(x = "number of decision trees",
+       y = "MSE on training data") +
+  theme_minimal(base_size = 33)
+
+combined_plot <- grid.arrange(p3, p1, p2, ncol=3, nrow=1, widths=c(1, 1, 1.35)) # so all plots are roughly the same width
+
+ggsave("../figure/bagging-mean.png", plot = combined_plot, width = 30, height = 8, dpi = 300)