read-in.Rmd

---
title: "Read raw data"
author: "Jens Daniel Müller"
date:  "`r format(Sys.time(), '%d %B, %Y')`"
output: 
  workflowr::wflow_html:
    number_sections: true
    toc_depth: 3
    toc_float:
      collapsed: false
editor_options:
  chunk_output_type: console
---


```{r global_options, include = FALSE}
knitr::opts_chunk$set(warning=FALSE, message=FALSE)
```


```{r packages}
library(tidyverse)
library(data.table)
library(lubridate)
library(DataExplorer)
library(leaflet)
library(readxl)
library(gsubfn)

```


```{r ggplot_theme, include = FALSE}
theme_set(theme_bw())
```

In this document, raw data files are read, merged into one file with harmonized column names and written as summarized data file.


# Sea-Bird SBE 16 sensor data (ts)

CTD sensor data including recordings from auxiliary pH, O~2~, Chla and pCO~2~ sensors were recorded with a measurement frequency of 15 sec. (In addition, pCO~2~ data were also internally recorded on the Contros HydroC instrument with higher tempral resolution and will later be used for further analysis after merging with CTD data.)

## Read regular profiles and transects


```{r read_ts_profile_transect_regular}

files <- list.files(path = "data/TinaV/Sensor/Profiles_Transects/", pattern = "[.]cnv$")
#file <- files[1]

for (file in files){
  
  start_date <- data.table(read.delim(here::here("data/TinaV/Sensor/Profiles_Transects/", file),
                                      sep="#", nrows = 160))[[78,1]]
  start_date <- substr(start_date, 15, 34)
  start_date <- mdy_hms(start_date, tz="UTC")
  
  temp <- read.delim(here::here("data/TinaV/Sensor/Profiles_Transects/", file),
                      sep="", skip = 160, header = FALSE)
  temp <- data.table(temp[,c(2,3,4,5,6,7,9,11,13)])
  names(temp) <- c("date_time", "dep", "tem", "sal", "V_pH", "pH", "Chl", "O2", "pCO2_analog")
  temp$start_date <- start_date
  temp$date_time <- temp$date_time + temp$start_date
  
  temp$ID <- substr(file, 1, 6)
  temp$type <- substr(file, 8,8)
  temp$station <- substr(file, 8,10)
  
  temp$cast <- "up"
  temp[date_time < mean(temp[dep == max(temp$dep)]$date_time)]$cast <- "down"
  
  if (exists("dataset")){
    dataset <- rbind(dataset, temp)
  }
  
  if (!exists("dataset")){
    dataset <- temp
  }
  
  rm(start_date)
  rm(temp)
}


ts <- dataset
rm(dataset, file, files)

```

## Read profiles and transects around Ostergarnsholm

```{r read_ts_profile_transect_Ostergarnsholm}

files <- list.files(path = "data/TinaV/Sensor/Ostergarnsholm/", pattern = "[.]cnv$")

for (file in files){
  
  start_date <- data.table(read.delim(here::here("data/TinaV/Sensor/Ostergarnsholm/", file),
                                      sep="#", nrows = 160))[[78,1]]
  start_date <- substr(start_date, 15, 34)
  start_date <- mdy_hms(start_date, tz="UTC")
  
  temp <- read.delim(here::here("data/TinaV/Sensor/Ostergarnsholm/", file),
                      sep="", skip = 160, header = FALSE)
  temp <- data.table(temp[,c(2,3,4,5,6,7,9,11,13)])
  names(temp) <- c("date_time", "dep", "tem", "sal", "V_pH", "pH", "Chl", "O2", "pCO2_analog")
  temp$start_date <- start_date
  temp$date_time <- temp$date_time + temp$start_date
  
  temp$ID <- substr(file, 1, 6)
  temp$type <- substr(file, 8,8)
  temp$station <- substr(file, 11,12)
    
  temp$cast <- "up"
  temp[date_time < mean(temp[dep == max(temp$dep)]$date_time)]$cast <- "down"
  
  if (exists("dataset")){
    dataset <- rbind(dataset, temp)
  }
  
  if (!exists("dataset")){
    dataset <- temp
  }
  
  rm(start_date)
  rm(temp)
}


ts_OGB <- dataset
rm(dataset, file, files)

ts_OGB <- ts_OGB %>% 
  mutate(type = if_else(station=="bo", "P", "T"),
         station = if_else(station == "bo", "P14", station),
         station = if_else(station == "in", "T14", station),
         station = if_else(station == "ou", "T15", station))

```

```{r merge_ts_regular_and_Ostergarnsholm}
ts <- bind_rows(ts, ts_OGB) %>% 
  arrange(date_time)

rm(ts_OGB)
```

## EDA raw data

```{r ts_EDA_raw, eval=FALSE}

source("code/eda.R")
eda(ts, "ts-raw")
rm(eda)

```

The output of an automated Exploratory Data Analysis (EDA) performed with the package `DataExplorer` can be accessed here:

[Link to EDA report of CTD raw data](https://jens-daniel-mueller.github.io/BloomSail/EDA_report_ts-raw.html){target="_blank}

## Clean data set

Sensor recordings were cleaned from obviously erroneous readings, by setting suspecious values to NA. 

```{r clean_ts_data}

class(ts)
ts <- data.table(ts)

# Profiling data

# temperature

# ts %>%
#   filter(type == "P") %>%
#   ggplot(aes(tem, dep, col=station, linetype = cast))+
#   geom_line()+
#   scale_y_reverse()+
#   geom_vline(xintercept = c(10, 20))+
#   facet_wrap(~ID)

ts[ID == "180723" & station == "P07" & dep < 2 & cast == "up"]$tem <- NA

# salinity

# ts %>%
#   filter(type == "P") %>%
#   ggplot(aes(sal, dep, col=station, linetype = cast))+
#   geom_path()+
#   scale_y_reverse()+
#   facet_wrap(~ID)

ts[sal < 6]$sal <- NA

# pH

# ts %>%
#   filter(type == "P") %>%
#   ggplot(aes(pH, dep, col=station, linetype=cast))+
#   geom_path()+
#   scale_y_reverse()+
#   facet_wrap(~ID)
# 
# ts %>%
#   filter(type == "P") %>%
#   ggplot(aes(V_pH, dep, col=station, linetype=cast))+
#   geom_path()+
#   scale_y_reverse()+
#   facet_wrap(~ID)

ts[pH < 7.5]$V_pH <- NA
ts[pH < 7.5]$pH <- NA

ts[ID == "180709" & station == "P03" & dep < 5 & cast == "down"]$pH <- NA
ts[ID == "180709" & station == "P05" & dep < 10 & cast == "down"]$pH <- NA
ts[ID == "180718" & station == "P10" & dep < 3 & cast == "down"]$pH <- NA
ts[ID == "180815" & station == "P03" & dep < 2 & cast == "down"]$pH <- NA
ts[ID == "180820" & station == "P11" & dep < 15 & cast == "down"]$pH <- NA

ts[ID == "180709" & station == "P03" & dep < 5 & cast == "down"]$V_pH <- NA
ts[ID == "180709" & station == "P05" & dep < 10 & cast == "down"]$V_pH <- NA
ts[ID == "180718" & station == "P10" & dep < 3 & cast == "down"]$V_pH <- NA
ts[ID == "180815" & station == "P03" & dep < 2 & cast == "down"]$V_pH <- NA
ts[ID == "180820" & station == "P11" & dep < 15 & cast == "down"]$V_pH <- NA


# pCO2

# ts %>%
#   filter(type == "P") %>%
#   ggplot(aes(pCO2, dep, col=station, linetype = cast))+
#   geom_path()+
#   scale_y_reverse()+
#   facet_wrap(~ID)

ts[ID == "180616"]$pCO2_analog <- NA

# O2

# ts %>% 
#   filter(type == "P") %>% 
#   ggplot(aes(O2, dep, col=station, linetype = cast))+
#   geom_path()+
#   scale_y_reverse()+
#   facet_wrap(~ID)

# Chlorophyll

# ts %>%
#   filter(type == "P") %>%
#   ggplot(aes(Chl, dep, col=station, linetype = cast))+
#   geom_path()+
#   scale_y_reverse()+
#   facet_wrap(~ID)

ts[Chl > 100]$Chl <- NA


#### Surface transect data

# ts %>%
#   filter(type == "T") %>%
#   ggplot(aes(date, dep, col=station))+
#   geom_point()+
#   scale_y_reverse()+
#   facet_wrap(~ID, scales = "free_x")
# 
# ts %>%
#   filter(type == "T") %>%
#   ggplot(aes(date, tem, col=station))+
#   geom_point()+
#   facet_wrap(~ID, scales = "free_x")
# 
# ts %>%
#   filter(type == "T") %>%
#   ggplot(aes(date, sal, col=station))+
#   geom_point()+
#   facet_wrap(~ID, scales = "free_x")
# 
# ts %>%
#   filter(type == "T") %>%
#   ggplot(aes(date, pCO2, col=station))+
#   geom_point()+
#   facet_wrap(~ID, scales = "free_x")
# 
# ts %>%
#   filter(type == "T") %>%
#   ggplot(aes(date, pH, col=station))+
#   geom_point()+
#   facet_wrap(~ID, scales = "free_x")
# 
# ts %>%
#   filter(type == "T") %>%
#   ggplot(aes(date, Chl, col=station))+
#   geom_point()+
#   facet_wrap(~ID, scales = "free_x")

ts[type == "T" & Chl > 10]$Chl <- NA


# ts %>% 
#   filter(type == "T") %>% 
#   ggplot(aes(date, O2, col=station))+
#   geom_point()+
#   facet_wrap(~ID, scales = "free_x")

```

## Write summary file

Relevant columns were selected and renamed, only observations from regular stations (P01-P13) and transects (T01-T13) were selected and summarized data were written to file.

```{r ts_column_subsetting_writing_summary_file, eval=FALSE}

ts <- ts %>% 
  select(date_time,
         ID,
         type,
         station,
         dep,
         sal,
         tem,
         pCO2_analog)

# ts <- ts %>% 
#   filter( !(station %in% c("PX1", "PX2", "TX1", "TX2") ))

ts %>% 
  write_csv(here::here("data/_summarized_data_files", "ts.csv"))

rm(ts)
```

## EDA clean data


```{r reopen_ts_data}
ts <- read_csv(here::here("data/_summarized_data_files", "ts.csv"),
               col_types = cols(pCO2_analog = col_double()))
```


```{r ts_EDA_clean, eval=FALSE}

source("code/eda.R")
eda(ts, "ts_clean")

rm(eda)
```


The output of an automated Exploratory Data Analysis (EDA) performed with the package `DataExplorer` can be accessed here:

[Link to EDA report of CTD clean data](https://jens-daniel-mueller.github.io/BloomSail/EDA_report_ts_clean.html){target="_blank}

## Overview plots

```{r temperature_profiles, fig.cap="temperature profiles by stations. Color refers to the starting date of each cruise.", fig.asp = 1.3}

ts %>%
  arrange(date_time) %>% 
  filter(type == "P", !(station %in% c("PX1", "PX2"))) %>%
  ggplot(aes(tem, dep, col=ymd(ID), group=ID))+
  geom_path()+
  scale_y_reverse()+
  scale_color_viridis_c(trans = "date", name="")+
  labs(x="temperature (°C)", y="Depth (m)")+
  facet_wrap(~station)

```


```{r pCO2_analog_profiles, fig.cap="pCO~2~ (analog signal) profiles by stations. Color refers to the starting date of each cruise.", fig.asp = 1.3}

ts %>%
  arrange(date_time) %>% 
  filter(type == "P", !(station %in% c("PX1", "PX2"))) %>%
  ggplot(aes(pCO2_analog, dep, col=ymd(ID), group=ID))+
  geom_path()+
  scale_y_reverse()+
  scale_color_viridis_c(trans = "date", name="")+
  labs(x="temperature (°C)", y="Depth (m)")+
  facet_wrap(~station)

```


# HydroC CO~2~ data (th)

## Read data

Originally, HydroC pCO~2~ data  were provided by KM Contros after applying a drift correction to the raw data, which was based on pre- and post-deployment calibration results. Those data are read-in here. However, later, the post-processing was repeated based on a cleaned data set.

```{r read_th_post_processed_by_manufacturer}

# Read Contros corrected data file, based on all recordings

th <-
  read_csv2(here::here("Data/TinaV/Sensor/HydroC-pCO2/corrected_Contros",
                       "parameter&pCO2s(method 43).txt"),
            col_names = c("date_time", "Zero", "Flush", "p_NDIR",
                          "p_in", "T_control", "T_gas", "%rH_gas",
                          "Signal_raw", "Signal_ref", "T_sensor",
                          "pCO2_corr", "Runtime", "nr.ave")) %>% 
  mutate(date_time = dmy_hms(date_time),
         Flush = as.factor(as.character(Flush)),
         Zero = as.factor(as.character(Zero)))

```

## Deployment identification and subsetting

Individual deployments (periods of observations with less than 30 sec between recordings) were identified and relevant deployment periods were subsetted. This procedure removes only recordings attributable to sensor testing and set-up.

```{r identify_select_th_deployment}

th <- th %>% 
  arrange(date_time) %>% 
  mutate(deployment = cumsum(c(TRUE,diff(date_time)>=30)))


th %>% 
  select(date_time, pCO2_corr, deployment) %>% 
  write_csv(here::here("Data/_summarized_data_files",
                       "th_all_data.csv"))

# th_out <- th %>% 
#   filter(!(deployment %in% c(2,6,9,14,17,21,23,27,31,33,34,35,37)))
# 
# th_out %>% 
#   filter(Zero == 1)

th <- th %>% 
  filter(deployment %in% c(2,6,9,14,17,21,23,27,31,33,34,35,37))

```

## Removal of duplicated time stamps

```{r remove_duplicated_timestamp_th}


# add counter for date_time observations

th <- th %>% 
  add_count(date_time)

# find triplicated time stamp and select only first observation, and merge

th_no_triple <- th %>% 
  filter(n <= 2)

th_triple_clean <- th %>% 
  filter(n > 2) %>% 
  slice(1)

th <- full_join(th_no_triple, th_triple_clean)

rm(list=setdiff(ls(), c("th")))


# find duplicated time stamps and shift first by one second backward, and merge

th %>% 
  distinct(date_time)

th <- th %>% 
  select(-n) %>% 
  add_count(date_time)

unique(th$n)

th_no_duplicated <- th %>%
  filter(n == 1)

th_duplicated <- th %>% 
  filter(n == 2)

th_duplicated_first <- th_duplicated %>% 
  group_by(date_time) %>% 
  slice(1) %>% 
  ungroup() %>% 
  mutate(date_time = date_time - 1)

th_duplicated_second <- th_duplicated %>% 
  group_by(date_time) %>% 
  slice(2) %>% 
  ungroup()

th_duplicated_clean <- full_join(th_duplicated_first, th_duplicated_second) %>% 
  arrange(date_time)

th <- full_join(th_no_duplicated, th_duplicated_clean)

th %>% 
  distinct(date_time)

rm(list=setdiff(ls(), c("th")))



# find duplicated time stamps and shift first by two seconds forward, and merge

th %>% 
  distinct(date_time)

th <- th %>% 
  select(-n) %>% 
  add_count(date_time)

unique(th$n)

th_no_duplicated <- th %>%
  filter(n == 1)

th_duplicated <- th %>% 
  filter(n == 2)

th_duplicated_first <- th_duplicated %>% 
  group_by(date_time) %>% 
  slice(1) %>% 
  ungroup() %>% 
  mutate(date_time = date_time + 2)

th_duplicated_second <- th_duplicated %>% 
  group_by(date_time) %>% 
  slice(2) %>% 
  ungroup()

th_duplicated_clean <- full_join(th_duplicated_first, th_duplicated_second) %>% 
  arrange(date_time)

th <- full_join(th_no_duplicated, th_duplicated_clean)

th %>% 
  distinct(date_time)

rm(list=setdiff(ls(), c("th")))

# remaining duplicates are observations where other observations with a +/- 1 sec timestamp exist
# for those cases, only the first duplicated observation is selected (similar to triplicate treatment)


th %>% 
  distinct(date_time)

th <- th %>% 
  select(-n) %>% 
  add_count(date_time)

unique(th$n)


th_still_no_duplicated <- th %>% 
  filter(n == 1)

th_still_duplicated_first <- th %>% 
  filter(n == 2) %>% 
  group_by(date_time) %>% 
  slice(1)

th <- full_join(th_still_no_duplicated, th_still_duplicated_first)

th %>% 
  distinct(date_time)

rm(list=setdiff(ls(), c("th")))

th <- th %>% 
  select(-n)

```

## Flush and Zeroing identification

```{r set_Flush_mixing_limits}

Flush_duration_lim <- 600
mixing_duration_lim <- 20

```


```{r Identify_Zeroing_Flush_th}

# Zeroing ID labeling

th <- th %>% 
  arrange(date_time) %>% 
  group_by(Zero) %>% 
  mutate(Zero_counter = as.factor(cumsum(c(TRUE,diff(date_time)>=30)))) %>% 
  ungroup()

unique(th$Zero_counter)

# Flush: Identification

th <- th %>% 
  mutate(Flush = 0) %>% 
  group_by(Zero, Zero_counter) %>% 
  mutate(start = min(date_time),
         duration = date_time - start,
         Flush = if_else(Zero == 0 & duration < Flush_duration_lim, "1", "0")) %>% 
  ungroup()


#  Flush: Identify equilibration and internal gas mixing periods

th <- th %>% 
  mutate(mixing = if_else(duration < mixing_duration_lim, "mixing", "equilibration"))

rm(Flush_duration_lim, mixing_duration_lim)

```

## Deployment plots

```{r th_deployment_plots, eval=FALSE}

pdf(file=here::here("output/Plots/read_in",
    "th_deployments.pdf"), onefile = TRUE, width = 7, height = 4)

for (i in unique(th$deployment)) {
  
  #i <- unique(th$deployment)[3]
  
  sub <-  th %>%
      filter(deployment == i)
  start_date <- min(sub$date_time)
  
  print(
    sub %>% 
      ggplot(aes(date_time, pCO2_corr, col=Zero_counter))+
      geom_line()+
      labs(title = paste("Deployment: ",i, "| Start time: ", start_date))
)
  
}

dev.off()

rm(sub, start_date, i)

```

A pdf with pCO~2~ timeseries plots of all deployments can be found here:

[Link to pCO~2~ timeseries plots](https://github.com/jens-daniel-mueller/BloomSail/tree/master/output/Plots/read_in/th_deployments.pdf){target="_blank"}


```{r th_EDA, eval=FALSE}

source("code/eda.R")
eda(th, "th")
rm(eda)

```

The output of an automated Exploratory Data Analysis (EDA) performed with the package `DataExplorer` can be accessed here:

[Link to EDA report of HydroC pCO~2~ data data](https://jens-daniel-mueller.github.io/BloomSail/EDA_report_th.html){target="_blank}

## Write summary file

Summarized pCO~2~ date were written to file.

```{r write_th_summary_file}

th %>% write_csv(here::here("Data/_summarized_data_files",
                            "th_full.csv"))

th %>% 
  select(date_time, Zero, Flush, pCO2_corr, deployment, Zero_counter, duration, mixing) %>% 
  write_csv(here::here("Data/_summarized_data_files",
                       "th.csv"))

rm(th)

```


# Bottle data CO2 (tb)

Discrete samples were collected with a Niskin bottle and analyzed for C[T] and A[T] at IOW CO~2~ lab.

## Read data

```{r read_tb_CO2_data, eval=FALSE}


tb <- read_csv(here::here("Data/TinaV/Bottle/Tracegases", "BloomSail_bottle_CO2_all.csv"),
                   col_types = list("c","c","n","n","n","n","n"))

tb <- tb %>% 
  select(ID=transect.ID,
         station=label,
         dep=Dep,
         sal=Sal,
         CT, AT)


```

## Write summary file

```{r write_tb_CO2_data, eval=FALSE}

tb %>% write_csv(here::here("Data/_summarized_data_files",
                            "tb.csv"))
rm(tb)

```


# Bottle data plankton (tp)

Discrete samples were collected with a Niskin bottle and analysed for phytoplankton composition and biomass at IOW CO~2~ lab.

## Read data

```{r read_tp_phytoplankton_data, eval=FALSE}


tp <- read_csv(here::here("Data/TinaV/Bottle/Phytoplankton",
                          "181205_BloomSail_Plankton_counts.csv"))

#### delete colomns that contain counts, not calculated biomass
tp <- tp[,-seq(4, 21,1)]

#### assign new column names
#### for species: nr = size class, HV = Heterocyst per Volume, Hl = Heterocyst per length, t = total

names(tp) <-
  c("date", "station", "dep",
    "Aphanizomenon.1",	"Aphanizomenon.2", "Aphanizomenon.3",
    "Aphanizomenon.t",	"Aphanizomenon.HV",	"Aphanizomenon.Hl",
    "Dolichospermum.1", "Dolichospermum.2", "Dolichospermum.3", "Dolichospermum.4",
    "Dolichospermum.t", "Dolichospermum.HV", "Dolichospermum.Hl",
    "Nodularia.1", "Nodularia.2", "Nodularia.3",
    "Nodularia.t", "Nodularia.HV", "Nodularia.Hl",
    "Nodulariadead.1", "Nodulariadead.2", "Nodulariadead.3",
    "Nodulariadead.t", 
    "total.t"
)


#### change format of data table and seperate into 2 columns for species and class

tp <- gather(tp, para, value, Aphanizomenon.1:total.t, factor_key=TRUE)
tp <- separate(tp, col = para, into = c("Species", "class"))

#### change class of columns
tp <- tp %>% 
  mutate(ID = date,
         date = ymd(date))

```

## Write summary file

```{r write_tp_phytoplankton_data, eval=FALSE}

tp %>% write_csv(here::here("Data/_summarized_data_files",
                            "tp.csv"))
rm(tp)

```



# GPS track (tt)

GPS track data were recorded with a Samsung Galaxy tablet.

## Read data

```{r read_tt_data, eval=FALSE}

files <- list.files(path = "data/TinaV/Track/GPS_Logger_Track/", pattern = "[.]txt$")


for (file in files){
  
  # if the merged dataset does exist, append to it
  if (exists("dataset")){
    
    temp<-data.table(read.delim(here::here("data/TinaV/Track/GPS_Logger_Track", file),
                                 sep=",")[,c(2,3,4)])
    names(temp) <- c("date_time", "lat", "lon")
    temp$date_time<- ymd_hms(temp$date, tz="UTC")
    
    dataset<-rbind(dataset, temp)
    rm(temp)
  }
  
  # if the merged dataset doesn't exist, create it
  if (!exists("dataset")){
    dataset<-data.table(read.delim(here::here("data/TinaV/Track/GPS_Logger_Track", file),
                                   sep=",")[,c(2,3,4)])
    names(dataset) <- c("date_time", "lat", "lon")
    dataset$date_time<- ymd_hms(dataset$date_time, tz="UTC")
    
  }
}

tt <- dataset
rm(dataset, file, files)

```

## Write summary file

```{r write_tt_data, eval=FALSE}

tt %>% 
  write_csv(here::here("data/_summarized_data_files",
                       "tt.csv"))

rm(tt)

```

# Ostergarnsholm

Atmospheric data were recorded at the ICOS station on Osterganrsholm.

## Read data

```{r read_wind_Ostergarn_tower, eval=FALSE}

og <- read_delim(here::here("Data/Ostergarnsholm/Tower", "Oes_Jens_atm_water_June_to_August_2018.csv"),
                 delim = ";"               )

og <- og %>%
  mutate(date_time = ymd_hms( paste(paste(year, month, day, sep = "/"),
                                          paste(hour, min, sec, sep = ":")))) %>% 
  select("date_time",
         "CO2 12m [ppm]",
         "w_c [ppm m/s]",
         "WS 12m [m/s]",
         "WD 12m [degrees]",
         "T 12m [degrees C]",
         "RIS [W/m^2]"
         )

og <- og %>% 
  select(date_time, pCO2_atm = "CO2 12m [ppm]", wind = "WS 12m [m/s]")

```

## Write summary file

```{r write_og_data, eval=FALSE}

og %>% 
  write_csv(here::here("data/_summarized_data_files",
                       "og.csv"))

rm(og)

```


# Finnmaid

pCO~2~ data were recorded on VOS Finnmaid in summer 2018.

## Read data

```{r read_fm_data, eval=FALSE}


### June - August 2018

files <- list.files(path = "data/Finnmaid_2018", pattern = "[.]xls$")
#file <-files[1]

for (file in files){
  
  
  temp <- read_excel(here::here("data/Finnmaid_2018", file))
  temp <- temp[c(1,2,3,12,7,4,15,8,5,17)]
  names(temp) <- c("date_time","lon","lat","pCO2","sal","tem","cO2","patm", "Teq","xCO2")
  temp <- temp[-c(1),]
  temp$date_time <- as.POSIXct(as.numeric(temp$date_time)*60*60*24, origin="1899-12-30", tz="GMT")
  temp$lon <- as.numeric(as.character(temp$lon))
  temp$lat <- as.numeric(as.character(temp$lat))
  temp$pCO2 <- as.numeric(as.character(temp$pCO2))
  temp$sal <- as.numeric(as.character(temp$sal))
  temp$tem <- as.numeric(as.character(temp$tem))
  temp$cO2 <- as.numeric(as.character(temp$cO2))
  temp$patm <- as.numeric(as.character(temp$patm))
  temp$Teq <- as.numeric(as.character(temp$Teq))
  temp$xCO2 <- as.numeric(as.character(temp$xCO2))
  temp <- data.table(temp)
  
  temp$route <- strapplyc(as.character(file), ".*(.).xls*", simplify = TRUE)
  temp$ID <- substr(as.character(file), 3, 10)
  
  if (exists("dataset")){
    dataset <- rbind (dataset, temp)
  } else{dataset <- temp}
  
}


rm(temp, files, file)
dataset <- dataset[pCO2 != 0]


#### Los Gatos data


files <- list.files(path = "data/Finnmaid_2018/LGR", pattern = "[.]xls$")
#file <-files[1]

for (file in files){
  
  
  temp <- read_excel(here::here("data/Finnmaid_2018/LGR", file))
  temp <- temp[c(2,3,4,8,6,5,14,7,15,9)]
  names(temp) <- c("date_time","lon","lat","pCO2","sal","tem","cO2","patm", "Teq","xCO2")
  temp <- temp[-c(1),]
  temp$date_time <- dmy_hms(temp$date_time)
  temp <- data.table(temp)
  
  temp$route <- substr(as.character(file), 12, 12)
  temp$ID <- substr(as.character(file), 3, 10)
  
  if (exists("dataset.LGR")){
    dataset.LGR <- rbind (dataset.LGR, temp)
  } else{dataset.LGR <- temp}
  
}

rm(temp, files, file)


```

## Convert O2 concentration units

```{r convert_O2_Los_Gatos_fm, eval=FALSE}

source(here::here("code", "O2stoO2c.R"))

dataset.LGR <- dataset.LGR %>%
  filter() %>% 
  mutate(cO2 = O2stoO2c(O2sat = cO2, T=tem, S=sal, P=3/10, p_atm = 1013.5))

rm(O2stoO2c, pH2Osat, sca_T, Scorr, TCorr, R, Vm)

```

```{r merge_Los_Gatos_LICOR_data_files, eval=FALSE}

dataset$sensor <- "LICOR"
dataset.LGR$sensor <- "LosGatos"

fm <- bind_rows(dataset, dataset.LGR)

rm(dataset, dataset.LGR, temp)


# dataset$Area <- with(dataset,
#                   ifelse(lon>12 & lon<12.6, "1.MEB",
#                   ifelse(lon>13.1 & lon<14.3, "2.ARK",
#                   ifelse(lat>57.5 & lat<58.5 & route %in% c("E", "G"), "4.EGS",
#                   ifelse(lat>57.3 & lat<57.5 & route %in% c("E"), "BS",
#                   ifelse(lat>56.8 & lat<57.5 & route=="W", "3.WGS",
#                   ifelse(lat>58.5 & lat<59 & lon>20, "5.NGS",
#                   ifelse(lon>22 & lon<24, "6.WGF",
#                   ifelse(lon>24 & lon<24.5, "7.HGF", "NaN")))))))))



```

## Write summary file

```{r write_fm, eval=FALSE}

# fm <- fm[complete.cases(fm[,pCO2]),]

fm %>% 
  write_csv(here::here("Data/_summarized_data_files",
                       "fm.csv"))
rm(fm)

```




# Interactive map

```{r map}

fm <- read_csv(here::here("Data/_summarized_data_files",
                       "fm.csv"))

fm_sub <- fm %>%
  arrange(date_time) %>% 
  slice(which(row_number() %% 20 == 1))

tt <- read_csv(here::here("Data/_summarized_data_files", "tt.csv"))

tt_sub <- tt %>%
  slice(which(row_number() %% 20 == 1))

rm(tt, fm)

leaflet() %>% 
  setView(lng = 20, lat = 57.3, zoom = 8) %>%
  addLayersControl(baseGroups = c("Ocean Basemap",
                                  "Satellite"),
                   overlayGroups = c("BloomSail", "Finnmaid"),
                   options = layersControlOptions(collapsed = FALSE),
                   position = 'topright') %>% 
  addProviderTiles("Esri.WorldImagery", group = "Satellite") %>%
  addProviderTiles(providers$Esri.OceanBasemap, group = "Ocean Basemap") %>%
  addScaleBar(position = 'topright') %>%
  addMeasure(
    primaryLengthUnit = "kilometers",
    secondaryLengthUnit = 'miles', 
    primaryAreaUnit = "sqmeters",
    secondaryAreaUnit="acres", 
    position = 'topleft') %>% 
  addCircles(data = fm_sub, ~lon, ~lat,
               color = "white",
               group = "Finnmaid") %>% 
  addPolylines(data = tt_sub, ~lon, ~lat,
               color = "red",
               group = "BloomSail")


rm(fm_sub, tt_sub)

```


# Tasks / open questions