Analyze_Results.Rmd

---
title: "ShinyLearner: Demo Analysis"
output: html_document
---

```{r setup, include=FALSE}
# Only need to run this if the packages have not been installed previously
#install.packages("tidyverse")
#install.packages("cowplot")
#setwd("~/Dropbox/ShinyLearner/Demo")
```

```{r}
library(tidyverse)
library(cowplot)
source("Helper.R")
```

## Data preparation

In the current working directory, we should have directories named "Results_Basic" and "Results_ParamsOptimized", which contain the results of the classification analysis with and without hyperparameter optimization, respectively (see Execute_Algorithms.ipynb). There should also be a directory called "Results_FeatureSelection," which contain results from executing the algorithms with feature selection. Each of these directories contains subdirectories labeled by dataset, each of which holds multiple tab-separated value (.tsv) files with the results.

For later use, we will create paths to the TSV files and store them in vectors. We will also build data frames with the results.

```{r}
data_dir_path <- "Datasets"
datasets <- sub(".tsv", "", list.files(data_dir_path, full.names = FALSE, recursive = FALSE))
```

```{r}
basic_time_file_paths = paste0("Results_Basic/", datasets, "/ElapsedTime.tsv")
basic_time <- NULL

for (path in basic_time_file_paths) {
  basic_time <- suppressMessages(read_tsv(path)) %>%
    mutate(Algorithm = paste0(basename(dirname(dirname(Algorithm))), "/", basename(dirname(Algorithm)))) %>%
    bind_rows(basic_time)
}

basic_time <- mutate(basic_time, Description = as.factor(Description)) %>%
  mutate(Algorithm  = as.factor(Algorithm))
```

```{r}
basic_metrics_file_paths = paste0("Results_Basic/", datasets, "/Metrics.tsv")
basic_metrics <- NULL

for (path in basic_metrics_file_paths) {
  basic_metrics <- suppressMessages(read_tsv(path)) %>%
    filter(Metric == 'AUROC') %>%
    select(-Metric) %>%
    mutate(Algorithm = paste0(basename(dirname(dirname(Algorithm))), "/", basename(dirname(Algorithm)))) %>%
    bind_rows(basic_metrics)
}

null_basic_metrics <- filter(basic_metrics, Description == "null") %>%
  dplyr::select(-Description) %>%
  mutate(Algorithm = as.factor(Algorithm)) %>%
  mutate(AnalysisType="Basic")

basic_metrics <- filter(basic_metrics, Description != "null") %>%
  mutate(Description = as.factor(Description)) %>%
  mutate(Algorithm = as.factor(Algorithm))
```

```{r}
optimized_metrics_file_paths = paste0("Results_ParamsOptimized/", datasets, "/Metrics.tsv")
optimized_metrics <- NULL

for (path in optimized_metrics_file_paths) {
  optimized_metrics <- suppressMessages(read_tsv(path)) %>%
    filter(Metric == 'AUROC') %>%
    select(-Metric) %>%
    mutate(Algorithm = paste0(basename(dirname(Algorithm)), "/", basename(Algorithm))) %>%
    bind_rows(optimized_metrics)
}

null_optimized_metrics <- filter(optimized_metrics, Description == "null") %>%
  dplyr::select(-Description) %>%
  mutate(Algorithm = as.factor(Algorithm)) %>%
  mutate(AnalysisType="Optimized hyperparameters")

optimized_metrics <- filter(optimized_metrics, Description != "null") %>%
  mutate(Description = as.factor(Description)) %>%
  mutate(Algorithm = as.factor(Algorithm))
```

```{r}
nested_metrics_file_paths = paste0("Results_ParamsOptimized/", datasets, "/Nested_Metrics.tsv")
nested_metrics <- NULL

for (path in nested_metrics_file_paths) {
  nested_metrics <- suppressMessages(read_tsv(path)) %>%
    filter(Description != "null") %>%
    filter(Metric == 'AUROC') %>%
    mutate(Params = paste0(basename(Algorithm))) %>%
    group_by(Description, Algorithm, Outer_Iteration, Params) %>%
    summarize(AUROC = median(Value)) %>%
    ungroup() %>%
    mutate(IsDefault = grepl("^default__", Params)) %>%
    mutate(Params=str_replace(Params, "default__", "")) %>%
    mutate(Params=str_replace(Params, "alt__", "")) %>%
    mutate(Params=str_replace_all(Params, "_", ", ")) %>%
    mutate(Algorithm=paste0(basename(dirname(dirname(Algorithm))), "/", basename(dirname(Algorithm)))) %>%
    dplyr::rename(Iteration = Outer_Iteration) %>%
    bind_rows(nested_metrics)
}

nested_metrics <- mutate(nested_metrics,
                         Description = as.factor(Description),
                         Algorithm = as.factor(Algorithm),
                         Iteration = as.factor(Iteration),
                         Params = as.factor(Params))
```

```{r}
predictions_data <- suppressMessages(read_tsv("Results_Basic/diabetes/Predictions.tsv")) %>%
  filter(Iteration==1) %>%
  dplyr::select(-Iteration) %>%
  mutate(Algorithm=factor(paste0(basename(dirname(dirname(Algorithm))), "/", basename(dirname(Algorithm))))) %>%
  mutate(Description=factor(Description)) %>%
  dplyr::select(-Description, -ActualClass, -PredictedClass, -InstanceID, -`1`) %>%
  dplyr::rename(Probability_of_Positive=`2`)
```

```{r}
optimized_predictions_data <- suppressMessages(read_tsv("Results_ParamsOptimized/diabetes/Nested_Predictions.tsv.gz")) %>%
  filter(Outer_Iteration==1 & Inner_Iteration == 1) %>%
  dplyr::select(-Outer_Iteration, -Inner_Iteration) %>%
  mutate(Params = paste0(basename(Algorithm))) %>%
  mutate(Algorithm=factor(paste0(basename(dirname(dirname(Algorithm))), "/", basename(dirname(Algorithm))))) %>%
  mutate(IsDefault = grepl("^default__", Params)) %>%
  mutate(Params=str_replace(Params, "default__", "")) %>%
  mutate(Params=str_replace(Params, "alt__", "")) %>%
  mutate(Params=str_replace_all(Params, "_", ", ")) %>%
  mutate(Params=factor(Params)) %>%
  mutate(InstanceID=factor(InstanceID)) %>%
  mutate(Description=factor(Description)) %>%
  dplyr::select(-Description, -ActualClass, -PredictedClass, -`1`) %>%
  dplyr::rename(Probability_of_Positive=`2`)
```

```{r}
fs_metrics_file_paths = paste0("Results_FeatureSelection/", datasets, "/Metrics.tsv")
fs_metrics = NULL

for (path in fs_metrics_file_paths) {
  fs_metrics <- suppressMessages(read_tsv(path)) %>%
    filter(Metric == 'AUROC') %>%
    select(-Metric) %>%
    mutate(Algorithm = paste0(basename(dirname(Algorithm)), "/", basename(Algorithm))) %>%
    bind_rows(fs_metrics)
}

null_fs_metrics <- filter(fs_metrics, Description == "null") %>%
  dplyr::select(-Description) %>%
  mutate(Algorithm = as.factor(Algorithm)) %>%
  mutate(AnalysisType="Feature selection")

fs_metrics <- filter(fs_metrics, Description != "null") %>%
  mutate(Description = as.factor(Description)) %>%
  mutate(Algorithm = as.factor(Algorithm))
```

```{r}
nested_fs_metrics_file_paths = paste0("Results_FeatureSelection/", datasets, "/Nested_Metrics.tsv")
nested_fs_metrics = NULL

# There's probably a tidier way to do this
for (path in nested_fs_metrics_file_paths) {
  tmp = suppressMessages(read_tsv(path)) %>%
    filter(Metric == 'AUROC') %>%
    dplyr::select(-Metric) %>%
    mutate(Feature_Selection_Algorithm=paste0(basename(dirname(dirname(Feature_Selection_Algorithm))), "/", basename(dirname(Feature_Selection_Algorithm)))) %>%
    mutate(Classification_Algorithm=paste0(basename(dirname(dirname(Classification_Algorithm))), "/", basename(dirname(Classification_Algorithm))))

  tmp2 = spread(tmp, Feature_Selection_Algorithm, Value)
  ranks = t(apply(tmp2, 1, function(x) { rank(x[(length(x) - 9):length(x)])}))
  tmp2[(ncol(tmp2) - 9):ncol(tmp2)] = ranks
  
  tmp2 = gather(tmp2, -Description, -Outer_Iteration, -Inner_Iteration, -Num_Features, -Classification_Algorithm, key=Feature_Selection_Algorithm, value=Value) %>%
    dplyr::rename(Rank=Value)

  nested_fs_metrics = dplyr::rename(tmp, AUROC=Value) %>%
    inner_join(tmp2) %>%
    bind_rows(nested_fs_metrics)
}

nested_fs_metrics <- filter(nested_fs_metrics, Description != "null")  %>%
  mutate(Description = as.factor(Description),
         Outer_Iteration = as.factor(Outer_Iteration),
         Inner_Iteration = as.factor(Inner_Iteration),
         Feature_Selection_Algorithm = as.factor(Feature_Selection_Algorithm),
         Num_Features = as.factor(Num_Features),
         Classification_Algorithm = as.factor(Classification_Algorithm))
```

```{r}
file_path = "Results_FeatureSelection/dermatology/Nested_SelectedFeatures_Summarized.tsv"

nested_feature_ranks <- suppressMessages(read_tsv(file_path)) %>%
  dplyr::rename(Algorithm=Feature_Selection_Algorithm) %>%
  mutate(Algorithm = factor(paste0(basename(dirname(dirname(Algorithm))), "/", basename(dirname(Algorithm))))) %>%
  mutate(Feature = factor(Feature))
```

## Prepare for plotting

```{r}
unlink("Figures", recursive = TRUE)
dir.create("Figures", showWarnings = FALSE)

unlink("Tables", recursive = TRUE)
dir.create("Tables", showWarnings = FALSE)

dimension_1 = 7
dimension_2 = 8

colorPalette = c("#a50026", "#d73027", "#f46d43", "#fdae61", "#fee090", "#abd9e9", "#74add1", "#4575b4", "#313695", "gray50")

axis_label_angle = 40
```

## Execution time per algorithm

```{r}
#plot_data <- group_by(basic_time, Algorithm) %>%
# summarize(ElapsedSeconds=median(ElapsedSeconds))

#make_bar_plot(plot_data,
#              x="Algorithm",
#              y="ElapsedSeconds",
#              xlab="Algorithm",
#              ylab="Elapsed seconds",
#              out_file_path="Figures/Datasets_Basic_ElapsedTime.pdf")

plot_data1 = group_by(basic_time, Algorithm, Iteration) %>%
 summarize(ElapsedSeconds=median(ElapsedSeconds))

plot_data2 = group_by(basic_metrics, Algorithm, Iteration) %>%
  summarize(AUROC=median(Value))

plot_data = inner_join(plot_data1, plot_data2, by = c("Algorithm", "Iteration"))

p = ggplot(plot_data, aes(x=ElapsedSeconds, y=AUROC)) +
  geom_point(size=5, aes(color=Algorithm)) +
  scale_color_manual(values=colorPalette, name="Classification algorithm") +
  xlab("Median execution time (seconds)") +
  ylab("Median AUROC") +
  theme_bw(base_size=14)

print(p)
  
ggsave("Figures/Datasets_Basic_ElapsedTime.pdf", width=dimension_2, height=dimension_1)
```

## AUROC values for each dataset (not hyperparameter optimized)

```{r}
plot_data <- group_by(basic_metrics, Description, Algorithm) %>%
  summarize(AUROC=median(Value))

make_faceted_point_plot(plot_data,
                        x="Algorithm",
                        y="AUROC",
                        ylab="AUROC (median per iteration)",
                        palette=colorPalette,
                        facet_column="Description",
                        out_file_path="Figures/Datasets_Basic_AUROC.pdf",
                        y_lines=c(0.5, 1),
                        ncol=2)
```

## Performance on null data

```{r}
plot_data <- bind_rows(null_basic_metrics, null_optimized_metrics, null_fs_metrics) %>%
  mutate(AnalysisType=factor(AnalysisType, levels=c("Basic", "Optimized hyperparameters", "Feature selection"))) %>%
  group_by(AnalysisType, Algorithm) %>%
  summarize(AUROC=median(Value))

make_faceted_point_plot(plot_data,
                        x="Algorithm",
                        y="AUROC",
                        ylab="AUROC (median per iteration)",
                        palette=colorPalette,
                        facet_column="AnalysisType",
                        out_file_path="Figures/Datasets_null_AUROC.pdf",
                        y_lines=c(0.5, 1),
                        ncol=1)
```

## Performance of each algorithm (not hyperparameter optimized)

```{r}
plot_data = group_by(basic_metrics, Description, Algorithm) %>%
  summarize(AUROC=median(Value)) %>%
  ungroup()

median_per_dataset <- group_by(plot_data, Description) %>%
  summarize(AUROC = median(AUROC)) %>%
  ungroup()

plot_data2 <- inner_join(plot_data, median_per_dataset, by="Description") %>%
  mutate(AUROC_Diff = AUROC.x - AUROC.y)
#  mutate(AUROC_Diff = (AUROC.x / AUROC.y - 1) * 100)

make_faceted_point_plot(plot_data2,
                        x="Description",
                        y="AUROC_Diff",
                        ylab="AUROC relative to median (per dataset)",
                        palette=colorPalette,
                        facet_column="Algorithm",
                        out_file_path="Figures/Algorithms_Basic_AUROC.pdf",
                        y_lines=0)

summary = dplyr::select(plot_data2, Algorithm, AUROC_Diff) %>%
  group_by(Algorithm) %>%
  summarize(Min_Diff=min(AUROC_Diff), Max_Diff=max(AUROC_Diff)) %>%
  ungroup()
write_tsv(summary, "Tables/Basic_DiffFromMedian.tsv")
```

## Consistency performance of "same" algorithm in different software packages (not hyperparameter optimized)

```{r}
make_algo_comparison_plot(basic_metrics,
                          algo1="mlr/mlp",
                          algo2="weka/MultilayerPerceptron",
                          out_file_path="Figures/Algorithms_Compare_MLP.pdf")

make_algo_comparison_plot(basic_metrics,
                          algo1="sklearn/logistic_regression",
                          algo2="weka/SimpleLogistic",
                          out_file_path="Figures/Algorithms_Compare_Logistic.pdf")
```

## Consistency of results across datasets for each algorithm (lower values = less variation)

```{r}
coef_var = function(x) {
  sd(x) / mean(x) * 100
}

plot_data <- group_by(basic_metrics, Algorithm, Description) %>%
  summarize(AUROC=median(Value)) %>%
  group_by(Algorithm) %>%
  summarize(AUROC_CV=coef_var(AUROC))

make_bar_plot(plot_data,
              x="Algorithm",
              y="AUROC_CV",
              xlab="Algorithm",
              ylab="Coefficient of variation (AUROC)",
              out_file_path="Figures/Algorithms_CoV_AUROC.pdf")
```

## Consistency of results across Monte Carlo iterations for each algorithm (lower values = less variation)

```{r}
plot_data <- group_by(basic_metrics, Algorithm, Description) %>%
  summarize(AUROC_CV=coef_var(Value)) %>%
  group_by(Algorithm) %>%
  summarize(AUROC_CV=median(AUROC_CV))

make_bar_plot(plot_data,
              x="Algorithm",
              y="AUROC_CV",
              xlab="Algorithm",
              ylab="Coefficient of variation (AUROC)",
              out_file_path="Figures/AlgorithmIterations_CoV_AUROC.pdf")
```

## Sample-level predictions for each algorithm

```{r}
ggplot(predictions_data, aes(Probability_of_Positive)) +
  geom_histogram(bins=40) +
  facet_wrap(~Algorithm, ncol=2) +
  xlab("Probabilistic prediction of positive diagnosis per patient") +
  ylab("Count") +
  theme_bw(base_size=18)

ggsave("Figures/Predictions_Histograms.pdf", width=dimension_1, height=dimension_2)
```

## Performance when optimizing vs. not optimizing hyperparameters

```{r}
plot_data <- inner_join(basic_metrics, optimized_metrics, by=c("Description", "Iteration", "Algorithm")) %>%
  mutate(AUROC_Diff = ((Value.y / Value.x) - 1) * 100) %>%
  group_by(Description, Algorithm) %>%
  summarize(AUROC_Diff=median(AUROC_Diff)) %>%
  ungroup() %>%
  mutate(Above_Zero=factor(AUROC_Diff > 0))

make_diff_bar_plot(plot_data,
                   x = "Description",
                   y = "AUROC_Diff",
                   fill_column = "Above_Zero",
                   facet_column = "Algorithm",
                   xlab = "Dataset",
                   ylab = "Percent change in AUROC (median per iteration)",
                   out_file_path = "Figures/Algorithms_ParamsImprovement_AUROC.pdf")

# make_faceted_point_plot(plot_data,
#                         x="Description",
#                         y="AUROC_Diff",
#                         ylab="Percent change in AUROC after hyperparameter optimization",
#                         palette=colorPalette,
#                         facet_column="Algorithm",
#                         out_file_path="Figures/Algorithms_ParamsImprovement_AUROC.pdf",
#                         y_lines=0)
```

## Performance of different hyperparameter combinations (for one algorithm) in nested cross-validation folds

```{r}
default_results <- filter(nested_metrics, Algorithm == "keras/dnn") %>%
  dplyr::select(-Algorithm) %>%
  filter(IsDefault) %>%
  dplyr::select(-IsDefault) %>%
  rename(Default_AUROC=AUROC)

alt_results <- filter(nested_metrics, Algorithm == "keras/dnn") %>%
  dplyr::select(-Algorithm) %>%
  filter(!IsDefault) %>%
  dplyr::select(-IsDefault)

plot_data <- inner_join(default_results, alt_results, by=c("Description", "Iteration")) %>%
  rename(Params=Params.y) %>%
  dplyr::select(-Params.x) %>%
  mutate(AUROC_Diff = AUROC - Default_AUROC) %>%
  dplyr::select(-AUROC, -Default_AUROC) %>%
  group_by(Description, Params) %>%
  summarize(AUROC_Diff=median(AUROC_Diff))

make_continuous_heatmap(plot_data,
                        x="Params",
                        y="Description",
                        fill_column = "AUROC_Diff",
                        xlabel="Hyperparameter combination",
                        ylabel="Dataset",
                        legend_title="",
                        out_file_path="Figures/Optimized_NestedMetrics_Heatmap.pdf",
                        keep_x_axis = FALSE)

write_tsv(spread(plot_data, key=Description, value=AUROC_Diff), "Tables/ParamOpt_Improvement.tsv")

plot_data2 <- separate(plot_data, Params, into=c("dropme", "dropout", "epochs", "layers", "regularization"), sep="=") %>%
  select(-dropme) %>%
  mutate(dropout=factor(str_replace(dropout, ", epochs", ""))) %>%
  mutate(epochs=factor(str_replace(epochs, ", layers", ""), levels=c(10, 50, 100))) %>%
  mutate(layers=str_replace(layers, ", regularization", "")) %>%
  mutate(layers=str_replace(layers, "16161616", "16x16x16x16")) %>%
  mutate(layers=str_replace(layers, "1616", "16x16")) %>%
  mutate(layers=str_replace(layers, "300100", "300x100")) %>%
  mutate(layers=factor(layers)) %>%
  mutate(regularization=factor(regularization))

y="AUROC_Diff"
ylab="AUROC (relative to default)"

A = make_param_box_plot(plot_data2, x="dropout", y=y, xlab="Dropout rate", ylab=ylab)
B = make_param_box_plot(plot_data2, x="epochs", y=y, xlab="# training epochs", ylab=ylab)
C = make_param_box_plot(plot_data2, x="layers", y=y, xlab="Layer structure", ylab=ylab)
D = make_param_box_plot(plot_data2, x="regularization", y=y, xlab="Regularization rate", ylab=ylab)

p = plot_grid(A, B, C, D, labels=c("A", "B", "C", "D"), label_size=16, ncol=2)
print(p)
ggsave("Figures/dnn_params.pdf", width=dimension_1, height=dimension_2)
```

## How much sample-level predictions change for different hyperparameters

```{r}
plot_data = filter(optimized_predictions_data, as.integer(InstanceID) < 4) %>%
  filter(Algorithm=="keras/dnn")

ggplot(plot_data, aes(x=Params, y=Probability_of_Positive, color=InstanceID, group=InstanceID)) +
  geom_point() +
  geom_line(alpha=0.3) +
  xlab("Hyperparameter combination") +
  ylab("Probabilistic prediction of positive diagnosis") +
  scale_color_manual(values=colorPalette[c(2, 4, 8)]) +
  guides(color=FALSE) +
  theme_bw(base_size=18) +
  theme(axis.text.x=element_blank(), axis.ticks.x=element_blank())

ggsave("Figures/Predictions_Hyperparams.pdf", width=dimension_1, height=dimension_2)
```

## Performance when performing feature selection vs. not

```{r}
plot_data <- inner_join(basic_metrics, fs_metrics, by=c("Description", "Iteration", "Algorithm")) %>%
  mutate(AUROC_Diff = ((Value.y / Value.x) - 1) * 100) %>%
  group_by(Description, Algorithm) %>%
  summarize(AUROC_Diff=median(AUROC_Diff)) %>%
  mutate(Above_Zero=factor(AUROC_Diff > 0))

make_diff_bar_plot(plot_data,
                   x = "Description",
                   y = "AUROC_Diff",
                   fill_column = "Above_Zero",
                   facet_column = "Algorithm",
                   xlab = "Dataset",
                   ylab = "Percent change in AUROC after feature selection",
                   out_file_path = "Figures/Algorithms_FSImprovement_AUROC.pdf")

# make_faceted_point_plot(plot_data,
#                         x="Description",
#                         y="AUROC_Diff",
#                         ylab="Percent change in AUROC after feature selection",
#                         palette=colorPalette,
#                         facet_column="Algorithm",
#                         out_file_path="Figures/Algorithms_FSImprovement_AUROC.pdf",
#                         y_lines=0)
```

## Performance for combinations of classification and feature-selection algorithms (higher average rank = better)

```{r}
plot_data = group_by(nested_fs_metrics, Description, Outer_Iteration, Feature_Selection_Algorithm, Num_Features, Classification_Algorithm) %>%
  summarize(Rank=mean(Rank), AUROC=mean(AUROC)) %>%
  group_by(Description, Feature_Selection_Algorithm, Num_Features, Classification_Algorithm) %>%
  summarize(Rank=mean(Rank), AUROC=mean(AUROC)) %>%
  group_by(Feature_Selection_Algorithm, Num_Features, Classification_Algorithm) %>%
  summarize(Rank=mean(Rank), AUROC=mean(AUROC)) %>%
  ungroup() %>%
  filter(Num_Features == 5) %>%
  dplyr::select(-Num_Features)

make_continuous_heatmap(plot_data,
                        x="Classification_Algorithm",
                        y="Feature_Selection_Algorithm",
                        fill_column = "Rank",
                        xlabel="Classification algorithm",
                        ylabel="Feature-selection algorithm",
                        legend_title="Average\nrank",
                        out_file_path="Figures/FS_vs_CL.pdf",
                        fill_values_range=NULL,
                        base_size=16)
```

## Performance of feature-selection algorithms by number of features

```{r}
plot_data = group_by(nested_fs_metrics, Description, Num_Features) %>%
  summarize(AUROC=mean(AUROC)) %>%
  ungroup()

min_max_scale = function(x) {(x - min(x)) / (max(x) - min(x))}

for (dataset in levels(plot_data$Description)) {
  matching_subset = filter(plot_data, Description==dataset)
  plot_data = filter(plot_data, Description != dataset)

  # We have to scale the data even after ranking it because different datasets
  #   have different numbers of features.
  matching_subset$AUROC = min_max_scale(pull(matching_subset, AUROC))
  plot_data = bind_rows(plot_data, matching_subset)
}

make_continuous_heatmap(plot_data,
                        x="Description",
                        y="Num_Features",
                        fill_column = "AUROC",
                        xlabel="Dataset",
                        ylabel="Number of features",
                        legend_title="",
                        out_file_path="Figures/FS_NumFeatures.pdf",
                        fill_values_range=NULL)
```

## Top features selected in one of the datasets

```{r}
feature_order = group_by(nested_feature_ranks, Feature) %>%
  summarize(Mean_Rank=median(Mean_Rank)) %>%
  ungroup() %>%
  arrange(Mean_Rank) %>%
  pull(Feature) %>%
  as.character()

plot_data = mutate(nested_feature_ranks, Feature = factor(as.character(Feature), levels=feature_order))

make_continuous_heatmap(plot_data,
                        x="Algorithm",
                        y="Feature",
                        fill_column = "Mean_Rank",
                        xlabel="Algorithm",
                        ylabel="Feature",
                        legend_title="Average\nrank",
                        out_file_path="Figures/FS_RankedFeatures.pdf",
                        fill_values_range=NULL,
                        base_size=11,
                        reverse_colors=TRUE)
```