eda.Rmd

---
title: "Exploratory data analysis"
output: 
  workflowr::wflow_html:
    includes:
      in_header: header.html
editor_options:
  chunk_output_type: console
author: "Patrick Schratz"
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(
  fig.retina = 3,
  fig.align = "center",
  fig.width = 6.93,
  fig.height = 6.13,
  out.width = "100%",
  echo = FALSE
)

source(here::here("code/99-packages.R"))
sourceDirectory("R")

# load drake objects
loadd(
  vi_data, nri_data, bands_data, coords_vi_nri_clean,
  
  fv_nri_car, fv_nri_info.gain, fv_nri_gain.ratio,
  fv_nri_symuncert, fv_nri_relief,
  fv_nri_rank, fv_nri_cor,
  fv_nri_mrmr,
  fv_nri_cmim, fv_nri_var,
  
  fv_vi_car, fv_vi_info.gain, fv_vi_gain.ratio,
  fv_vi_symuncert, fv_vi_relief,
  fv_vi_rank, fv_vi_cor,
  fv_vi_mrmr,
  fv_vi_cmim, fv_vi_var,
  
  fv_hr_car, fv_hr_info.gain, fv_hr_gain.ratio,
  fv_hr_symuncert, fv_hr_relief,
  fv_hr_rank, fv_hr_cor,
  fv_hr_mrmr,
  fv_hr_cmim, fv_hr_var,
  
  
  fv_hr_info.gain5, fv_hr_info.gain10,
  fv_hr_info.gain15, fv_hr_info.gain20,
  fv_hr_info.gain25, fv_hr_info.gain30,
  
  fv_vi_info.gain5, fv_vi_info.gain10,
  fv_vi_info.gain15, fv_vi_info.gain20,
  fv_vi_info.gain25, fv_vi_info.gain30,
  
  fv_nri_info.gain5, fv_nri_info.gain10,
  fv_nri_info.gain15, fv_nri_info.gain20,
  fv_nri_info.gain25, fv_nri_info.gain30
)
library("DataExplorer")
library("dplyr")
library("ggsci")
library("ggplot2")
library("rgdal")
```

# Datasets {.tabset .tabset-fade}

## VI

### Overview

```{r eda-1 }
intro <- introduce(vi_data)
intro_df <- data.frame(
  "Name" = c(
    "Rows", "Columns",
    "Discrete columns", "Continuous columns", "All missing columns",
    "Missing observations", "Complete Rows",
    "Total observations"
  ),
  "Value" = c(
    format(intro[["rows"]], big.mark = ","),
    format(intro[["columns"]], big.mark = ","),
    format(intro[["discrete_columns"]], big.mark = ","),
    format(intro[["continuous_columns"]], big.mark = ","),
    format(intro[["all_missing_columns"]], big.mark = ","),
    format(intro[["total_missing_values"]], big.mark = ","),
    format(intro[["complete_rows"]], big.mark = ","),
    format(intro[["total_observations"]], big.mark = ",")
  )
)
knitr::kable(intro_df)
```

### Histograms

```{r eda-2 }
plot_histogram(vi_data)
```

### PCA

```{r eda-3 }
plot_prcomp(vi_data)
```

### Corr

```{r eda-4 }
plot_correlation(vi_data)
```

## NRI

### Overview

```{r eda-5 }
intro <- introduce(nri_data)
intro_df <- data.frame(
  "Name" = c(
    "Rows", "Columns",
    "Discrete columns", "Continuous columns", "All missing columns",
    "Missing observations", "Complete Rows",
    "Total observations"
  ),
  "Value" = c(
    format(intro[["rows"]], big.mark = ","),
    format(intro[["columns"]], big.mark = ","),
    format(intro[["discrete_columns"]], big.mark = ","),
    format(intro[["continuous_columns"]], big.mark = ","),
    format(intro[["all_missing_columns"]], big.mark = ","),
    format(intro[["total_missing_values"]], big.mark = ","),
    format(intro[["complete_rows"]], big.mark = ","),
    format(intro[["total_observations"]], big.mark = ",")
  )
)
kable(intro_df)
```

### Histograms

No histograms for NRI -> too many features.

### PCA

```{r eda-6 }
plot_prcomp(nri_data)
```

### Corr

```{r eda-7 }
plot_correlation(nri_data)
```

## HR

### Overview

```{r eda-8 }
intro <- introduce(bands_data)
intro_df <- data.frame(
  "Name" = c(
    "Rows", "Columns",
    "Discrete columns", "Continuous columns", "All missing columns",
    "Missing observations", "Complete Rows",
    "Total observations"
  ),
  "Value" = c(
    format(intro[["rows"]], big.mark = ","),
    format(intro[["columns"]], big.mark = ","),
    format(intro[["discrete_columns"]], big.mark = ","),
    format(intro[["continuous_columns"]], big.mark = ","),
    format(intro[["all_missing_columns"]], big.mark = ","),
    format(intro[["total_missing_values"]], big.mark = ","),
    format(intro[["complete_rows"]], big.mark = ","),
    format(intro[["total_observations"]], big.mark = ",")
  )
)
kable(intro_df)
```

### Histograms

```{r eda-9 }
plot_histogram(bands_data)
```

### PCA

```{r eda-10 }
plot_prcomp(bands_data)
```

### Corr

```{r eda-11 }
plot_correlation(bands_data)
```

# Custom plots

```{r eda-12 }
vi_data_plot <- vi_data %>%
  mutate(plot = factor(rep(
    c("Laukiz 1", "Laukiz 2", "Luiando", "Oiartzun"),
    c(479, 451, 300, 529)
  )))
```

# Mean defoliation per plot

```{r eda-13 }
mean_defol <- vi_data_plot %>%
  group_by(plot) %>%
  summarise(mean(defoliation)) %>%
  pull(.)
vi_data_plot %>%
  group_by(plot) %>%
  summarise(mean(defoliation))
```

# Coefficient of variation

```{r eda-14 }
cov_defol <- vi_data_plot %>%
  group_by(plot) %>%
  summarise((sd(defoliation) / mean(defoliation)) * 100) %>%
  pull(.)
vi_data_plot %>%
  group_by(plot) %>%
  summarise((sd(defoliation) / mean(defoliation)) * 100)
```

# sd / skewness

```{r eda-15 }
sd_skewness_defol <- vi_data_plot %>%
  group_by(plot) %>%
  summarise(((sd(defoliation) / mean(defoliation)) * 100) / e1071::skewness(defoliation)) %>%
  pull(.)
vi_data_plot %>%
  group_by(plot) %>%
  summarise(((sd(defoliation) / mean(defoliation)) * 100) / e1071::skewness(defoliation))
```

```{r eda-16 }
sd_defol <- vi_data_plot %>%
  group_by(plot) %>%
  summarise(sd(defoliation)) %>%
  pull(.)
vi_data_plot %>%
  group_by(plot) %>%
  summarise(sd(defoliation))
```

```{r defoliation-distribution-plot, dev = c("png", "pdf")}
boxplot_defol <- vi_data_plot %>%
  group_by(plot) %>%
  ggboxplot(
    x = "plot", y = "defoliation", color = "plot",
    add = "jitter", add.params = list(size = "defoliation")
  ) +
  scale_size(range = c(0.5, 0.5)) +
  annotate("text",
           label = expression(bold(atop("n = 479", bar(x) ~ "= 55.96"))), x = 1,
           y = 112, size = 3, colour = "#BC3C29", fontface = 2
  ) +
  annotate("text",
           label = expression(bold(atop("n = 451", bar(x) ~ "= 13.54"))), x = 2,
           y = 112, size = 3, colour = "#0072B5", fontface = 2
  ) +
  annotate("text",
           label = expression(bold(atop("n = 300", bar(x) ~ "= 68.44"))), x = 3,
           y = 112, size = 3, colour = "#E18727", fontface = 2
  ) +
  annotate("text",
           label = expression(bold(atop("n = 529", bar(x) ~ "= 69.22"))), x = 4,
           y = 112, size = 3, colour = "#20854E", fontface = 2
  ) +
  scale_color_nejm() +
  theme(legend.position = "none") +
  labs(y = "Total defoliation per tree (%)", x = "Plot")
boxplot_defol
```

# Point density

In Meters.

```{r eda-18 }
plots <- list("Laukiz 1", "Laukiz 2", "Luiando", "Oiartzun")
dist <- map(plots, ~ {
  coords <- coords_vi_nri_clean %>%
    mutate(plot = factor(rep(
      c("Laukiz 1", "Laukiz 2", "Luiando", "Oiartzun"),
      c(479, 451, 300, 529)
    ))) %>%
    filter(plot == .x) %>%
    dplyr::select(-plot)
  points <- SpatialPoints(
    coords = coords,
    proj4string = CRS("+proj=utm +zone=30 +ellps=WGS84 +datum=WGS84 +units=m +no_defs")
  )
  distance <- mean(as.dist(pointDistance(points,
                                         allpairs = TRUE,
                                         lonlat = FALSE
  )))
})
set_names(dist, plots)
dist_plots <- unlist(dist)
```

```{r eda-19, echo = FALSE}
# file_move("docs/figure/eda.Rmd/defoliation-distribution-plot-1.pdf",
#           "code/98-paper/journal/")
```

# Correlation of filter methods {.tabset .tabset-fade}

## VI

### Filter methods amongst each other

```{r eda-20}
# combine filter values

fv_all <- dplyr::bind_rows(
  fv_vi_car$data,
  fv_vi_info.gain$data,
  fv_vi_gain.ratio$data,
  fv_vi_symuncert$data,
  fv_vi_relief$data,
  fv_vi_rank$data,
  fv_vi_cor$data,
  fv_vi_mrmr$data,
  fv_vi_cmim$data,
  fv_vi_var$data
) %>%
  dplyr::select(value, method) %>%
  mutate(id = rep(1:89, 10)) %>%
  tidyr::spread(method, value) %>%
  mutate_if(is.numeric, list(~ na_if(., -Inf))) %>% # Inf in mrmr
  na.omit() %>% # NA in cmim
  dplyr::select(-id)


corr <- cor(fv_all, method = "spearman")
# p.mat = cor_pmat(corr)

ggcorrplot(corr,
           # p.mat = p.mat,
           hc.order = F,
           type = "lower",
           lab = TRUE
)
```

### Number of bins of `FSelectorRcpp::information.gain()`

```{r}
fv_all <- dplyr::bind_rows(
  fv_vi_info.gain5$data, fv_vi_info.gain10$data,
  fv_vi_info.gain15$data, fv_vi_info.gain20$data,
  fv_vi_info.gain25$data, fv_vi_info.gain30$data
)  %>%
  dplyr::select(value, method) %>%
  mutate(id = rep(1:89, 10)) %>%
  tidyr::spread(method, value) %>%
  mutate_if(is.numeric, list(~ na_if(., -Inf))) %>% # Inf in mrmr
  na.omit() %>% 
  dplyr::select(-id)

corr <- cor(fv_all, method = "spearman")

ggcorrplot(corr,
           # p.mat = p.mat,
           hc.order = F,
           type = "lower",
           lab = TRUE
)
```

## NRI

```{r eda-21}
# combine filter values

fv_all <- dplyr::bind_rows(
  fv_nri_car$data,
  fv_nri_info.gain$data,
  fv_nri_gain.ratio$data,
  fv_nri_symuncert$data,
  fv_nri_relief$data,
  fv_nri_rank$data,
  fv_nri_cor$data,
  fv_nri_mrmr$data,
  fv_nri_cmim$data,
  fv_nri_var$data
) %>%
  dplyr::select(value, method) %>%
  mutate(id = rep(1:7381, 10)) %>%
  tidyr::spread(method, value) %>%
  dplyr::select(-id)


corr <- cor(fv_all, method = "spearman")
p.mat <- cor_pmat(corr)

ggcorrplot(corr,
           # p.mat = p.mat,
           hc.order = FALSE,
           type = "lower",
           lab = TRUE
)
```

## HR

```{r eda-22}
# combine filter values

fv_all <- dplyr::bind_rows(
  fv_hr_car$data,
  fv_hr_info.gain$data,
  fv_hr_gain.ratio$data,
  fv_hr_symuncert$data,
  fv_hr_relief$data,
  fv_hr_rank$data,
  fv_hr_cor$data,
  fv_hr_mrmr$data,
  fv_hr_cmim$data,
  fv_hr_var$data
) %>%
  dplyr::select(value, method) %>%
  mutate(id = rep(1:122, 10)) %>%
  tidyr::spread(method, value) %>%
  dplyr::select(-id)


corr <- cor(fv_all, method = "spearman")
p.mat <- cor_pmat(corr)

ggcorrplot(corr,
           # p.mat = p.mat,
           hc.order = F,
           type = "lower",
           lab = TRUE
)
```