Merge pull request #28 from dafenner/develop

Update to SRTM download, bug fix in cqcp_check_input, documentation update

DESCRIPTION

@@ -1,8 +1,8 @@
 Package: CrowdQCplus
 Type: Package
 Title: Enhanced quality control for crowdsourced data from citizen weather stations
-Version: 1.0.0
-Date: 2021-11-01
+Version: 1.0.1
+Date: 2022-09-08
 Author: Daniel Fenner, Tom Grassmann, Benjamin Bechtel, Matthias Demuzere, Jonas Kittner, Fred Meier
 Authors@R: c(
     person("Daniel", "Fenner", email = "daniel.fenner@meteo.uni-freiburg.de", role = c("aut", "cre")),
@@ -14,13 +14,13 @@ Authors@R: c(
 Maintainer: Daniel Fenner <daniel.fenner@meteo.uni-freiburg.de>
 Description: This package performs a quality control (QC) and filters suspicious data from citizen weather stations (CWS). It was originally designed for and tested with hourly air-temperature data but should work with other near-normally distributed data. It is based on the package 'CrowdQC' but offers several additions and improvements.
 License: GPL-3 + file LICENCE
-Citation: Fenner et al. 2021
+Citation: Fenner et al. 2022
 URL: https://github.com/dafenner/CrowdQCplus
 BugReports: https://github.com/dafenner/CrowdQCplus/issues
 Encoding: UTF-8
 LazyData: true
-RoxygenNote: 7.1.2
+RoxygenNote: 7.2.1
 Depends: R (>= 3.5.0)
-Imports: data.table, methods, stats, robustbase, lubridate, sp, raster, rgdal
+Imports: data.table, methods, stats, robustbase, lubridate, sp, raster, rgdal, geodata
 NeedsCompilation: no
-Packaged: 2021-11-01 22:50 UTC; dafenner
+Packaged: 2022-09-08 12:45 UTC; dafenner

R/cqcp_filter.R

@@ -1,5 +1,5 @@
 # CrowdQC+ - Quality control for citizen weather station data.
-# Copyright (C) 2021  Daniel Fenner, Tom Grassmann, Benjamin Bechtel, Matthias Demuzere, Jonas Kittner, Fred Meier
+# Copyright (C) 2022  Daniel Fenner, Tom Grassmann, Benjamin Bechtel, Matthias Demuzere, Jonas Kittner, Fred Meier
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -563,8 +563,23 @@ cqcp_o3 <- function(data, cutOff = 0.8, complete = FALSE, duration = NULL,
 #' In the correction the original data "ta" is used instead of the interpolated
 #' values "ta_int". Hence, this function can be applied at/after any QC level. 
 #' Diverging from all other QC levels, no additional flag variable with TRUE/FALSE 
-#' is added to the data.table during cqcp_o4. Data after the correction thus can be 
+#' is added to the data.table during cqcp_o4. Corrected data can thus be 
 #' selected at any QC level. The corrected data are written in a new column 'ta_corr'. 
+#' For Netatmo CWS (https://www.netatmo.com/en-us/weather), Coney et al. (2022) 
+#' determined a mean time constant of the air-temperature sensor of 
+#' tau = 12.7 minutes (762 seconds).
+#' Büchau (2018) determined a mean time constant of the Netatmo air-temperature 
+#' sensor (including silver aluminium shell) of tau = 24.675 minutes (1480.5 seconds).
+#' 
+#' References: 
+#' Büchau, Y. G. (2018): Modelling Shielded Temperature Sensors - An Assessment 
+#' of the Netatmo Citizen Weather Station. MSc Thesis, Universität Hamburg, Germany.
+#' Available at: https://bis-erdsystem.de/fileadmin/user_upload/bise/Texte/MSC_Met_61-20180313.pdf
+#' 
+#' Coney, J., Pickering, B., Dufton, D., Lukach, M., Brooks, B. and Neely, R. R. (2022):
+#' How useful are crowdsourced air temperature observations? An assessment of 
+#' Netatmo stations and quality control schemes over the United Kingdom. 
+#' Meteorol. Appl. 29 (3): e2075. https://doi.org/10.1002/met.2075
 #'
 #' @param data data.table in CrowdQC+ format (columns "time" and "ta" must be present)
 #' @param time_constant Time constant value for the sensor in seconds (must be 

R/cqcp_helper.R

@@ -1,5 +1,5 @@
 # CrowdQC+ - Quality control for citizen weather station data.
-# Copyright (C) 2021  Daniel Fenner, Tom Grassmann, Benjamin Bechtel, Matthias Demuzere, Jonas Kittner, Fred Meier
+# Copyright (C) 2022  Daniel Fenner, Tom Grassmann, Benjamin Bechtel, Matthias Demuzere, Jonas Kittner, Fred Meier
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -73,7 +73,7 @@ cqcp_check_input <- function(data, print = TRUE, file = NULL, as_list = FALSE){
     if(!has_ta) miss_1a <- c(miss_1a, "ta")
     if(!has_lon) miss_1a <- c(miss_1a, "lon")
     if(!has_lat) miss_1a <- c(miss_1a, "lat")
-    mess_1a <- cqcp_colourise(paste0("     ! Missing: ",paste(test, collapse = ", "),"\n     --> CrowdQC+ will not work with this data.\n"), "red")
+    mess_1a <- cqcp_colourise(paste0("     ! Missing: ", paste(miss_1a, collapse = ", "),"\n     --> CrowdQC+ will not work with this data.\n"), "red")
     ch_1a <- FALSE
     ok <- FALSE
   } else mess_1a <- cqcp_colourise("     OK\n", "green")
@@ -83,7 +83,7 @@ cqcp_check_input <- function(data, print = TRUE, file = NULL, as_list = FALSE){
   if(!has_z) {
     miss_1b <- c(miss_1b, "z")
     mess_1b <- cqcp_colourise("     ! Missing: z\n", "yellow")
-    mess_1b <- cqcp_colourise("     --> Filters cqcp_m2 and cqcp_m5 will not work with 'heightCorrection = T'. You can run 'cqcp_add_dem_height' to add DEM information.\n", "yellow")
+    mess_1b <- c(mess_1b, cqcp_colourise("     --> Filters cqcp_m2 and cqcp_m5 will not work with 'heightCorrection = T'. You can run 'cqcp_add_dem_height' to add DEM information.\n", "yellow"))
     ch_1b <- FALSE
   } else mess_1b <- cqcp_colourise("     OK\n", "green")
 
@@ -262,7 +262,7 @@ cqcp_extract_raster_data <- function(lon, lat, raster = NULL, file = NULL){
   return(value) # Output
 }
 
-#' Download SRTM data using raster package for bounding box of data.
+#' Download SRTM data using geodata package for bounding box of data.
 #' 
 #' SRTM data is automatically downloaded and, in case of more than one SRTM tile, 
 #' merged together as a mosaic.
@@ -276,7 +276,7 @@ cqcp_extract_raster_data <- function(lon, lat, raster = NULL, file = NULL){
 #' @param outfile File path to save the SRTM raster as geotiff 
 #' @param overwrite Overwrite existing geotiff? Default is TRUE. 
 #' @param crop Crop raster/geotiff to data extent? Default is FALSE. 
-#' @param ... Additional parameters supported by raster::getData
+#' @param ... Additional parameters supported by geodata::elevation_3s
 #'
 #' @return RasterLayer object with SRTM data
 #' @export
@@ -286,7 +286,7 @@ cqcp_download_srtm <- function(data, directory = NULL, outfile = NULL,
   # Check if directory exists
   if(!is.null(directory)) {
     if(!dir.exists(directory)) dir.create(directory)
-  }
+  } else directory = getwd()
 
   # Get bounding box coordinates
   min_lon <- min(data$lon)
@@ -295,10 +295,10 @@ cqcp_download_srtm <- function(data, directory = NULL, outfile = NULL,
   max_lat <- max(data$lat)
 
   # Download data
-  ll <- raster::getData(name="SRTM", lat=min_lat, lon=min_lon, path = directory, ...)
-  lr <- raster::getData(name="SRTM", lat=min_lat, lon=max_lon, path = directory, ...)
-  ur <- raster::getData(name="SRTM", lat=max_lat, lon=max_lon, path = directory, ...)
-  ul <- raster::getData(name="SRTM", lat=max_lat, lon=min_lon, path = directory, ...)
+  ll <- raster::raster(geodata::elevation_3s(lat=min_lat, lon=min_lon, path = directory, ...))
+  lr <- raster::raster(geodata::elevation_3s(lat=min_lat, lon=max_lon, path = directory, ...))
+  ur <- raster::raster(geodata::elevation_3s(lat=max_lat, lon=max_lon, path = directory, ...))
+  ul <- raster::raster(geodata::elevation_3s(lat=max_lat, lon=min_lon, path = directory, ...))
 
   # Combine tiles, if necessary
   mosaic <- raster::mosaic(ll, lr, ur, ul, fun=mean)
@@ -313,7 +313,7 @@ cqcp_download_srtm <- function(data, directory = NULL, outfile = NULL,
   # Store merged raster?
   if(!is.null(outfile)) {
     if(!dir.exists(dirname(outfile))) dir.create(dirname(outfile))
-    raster::writeRaster(mosaic, filename=outfile, overwrite=overwrite)
+    raster::writeRaster(mosaic, filename = outfile, overwrite = overwrite)
   }
 
   return(mosaic)
@@ -349,7 +349,7 @@ cqcp_download_srtm <- function(data, directory = NULL, outfile = NULL,
 #' @param na_vals Set NA values in DEM to this value to avoid missing value in 
 #'   cqcp_m2. Default: 0
 #' @param quiet Suppress messages by CrowdQC+. Default: FALSE
-#' @param ... Additional parameters supported by raster::getData
+#' @param ... Additional parameters supported by geodata::elevation_3s
 #'
 #' @return data table with new column 'z' with DEM information
 #' @export

README.md

@@ -1,10 +1,10 @@
 # CrowdQC+
 
-This R package performs a quality control (QC) and filters suspicious data from citizen weather stations (CWS). It is based on the package <a href="http://dx.doi.org/10.14279/depositonce-6740.3">'CrowdQC'</a> but offers several additions, improvements, and bug fixes. Both packages were originally designed for and tested with air-temperature data but should also work with other near-normally distributed data. It is not designed for precipitation data.
+This R package performs a quality control (QC) and filters suspicious data from citizen weather stations (CWS). It is based on the package ['CrowdQC'](http://dx.doi.org/10.14279/depositonce-6740.3) but offers several additions, improvements, and bug fixes. Both packages were originally designed for and tested with air-temperature data but should also work with other near-normally distributed data. It is not designed for precipitation data.
 
 CrowdQC+ is a statistically-based QC that identifies individual possibly faulty observations by comparing them to a large crowd of other observations. It does not need reference meteorological observations to be applied. The main idea of CrowdQC+ is that there is trustworthy information in the crowd, which can be used to check and remove individual values during the QC. Yet, there is no guarantee that all faulty observations are filtered by the QC.
 
-A detailed description of the functionalities and an evaluation of the performance of the QC can be found in this open-access journal article: CrowdQC+ – A quality-control for crowdsourced air-temperature observations enabling world-wide urban climate applications. Frontiers in Environmental Science.<br>
+A detailed description of the functionalities and an evaluation of the performance of the QC can be found in this open-access journal article: [CrowdQC+ – A quality-control for crowdsourced air-temperature observations enabling world-wide urban climate applications. Frontiers in Environmental Science](https://doi.org/10.3389/fenvs.2021.720747).<br>
 It is recommended that you read the article before you start working with CrowdQC+ to understand its capabilities and limitations.
 
 ## Dependencies
@@ -39,7 +39,7 @@ install.packages("rgdal", configure.args = c(rgdal = "--with-proj_api=proj_api.h
 
 **Option 1:**
 
-Directly pull the code from this repository into your programming environment, using the <a href="https://devtools.r-lib.org/">devtools</a> package:
+Directly pull the code from this repository into your programming environment, using the [devtools](https://devtools.r-lib.org/) package:
 
 ```
 install.packages("devtools")
@@ -48,15 +48,15 @@ devtools::install_github("dafenner/CrowdQCplus")
 
 **Option 2:**
 
-Download the <a href="https://github.com/dafenner/CrowdQCplus/archive/refs/heads/master.zip">zip-file</a> from this repository, save it locally, and install it in your programming environment using the <a href="https://devtools.r-lib.org/">devtools</a> package:
+Download the [zip-file](https://github.com/dafenner/CrowdQCplus/archive/refs/heads/master.zip) from this repository, save it locally, and install it in your programming environment using the [devtools](https://devtools.r-lib.org/) package:
 ```
 install.packages("devtools")
 devtools::install_local(<PATH_TO_THE_ZIP_FILE>)
 ```
 
 **Option 3:**
 
-Download the latest release of CrowdQC+ as a `.tar.gz` file <a href="https://github.com/dafenner/CrowdQCplus/releases"> (list or releases)</a>, save it locally, and install it in your programming environment:
+Download the latest release of CrowdQC+ as a `.tar.gz` file ([list of releases](https://github.com/dafenner/CrowdQCplus/releases)), save it locally, and install it in your programming environment:
 ```
 install.packages(<PATH_TO_THE_tar.gz_FILE>, repos = NULL, type ="source")
 ```
@@ -70,7 +70,7 @@ library(CrowdQCplus)
 
 ## Using CrowdQC+
 ### Data
-Data should be represented as a <a href="https://CRAN.R-project.org/package=data.table">data.table</a> with the following required columns:
+Data should be represented as a [data.table](https://CRAN.R-project.org/package=data.table) with the following required columns:
 
 `p_id`: unique ID of each station<br>
 `time`: time as POSIX.ct. Keep in mind time zones!<br>
@@ -127,19 +127,19 @@ if(ok) {
 ```
 
 ## How to contribute?
-If you are using CrowdQC+ and have ideas how to make it better, improve its performance, resolve errors, please create <a href="https://github.com/dafenner/CrowdQCplus/issues">issues</a>.
+If you are using CrowdQC+ and have ideas how to make it better, improve its performance, resolve errors, please create [issues](https://github.com/dafenner/CrowdQCplus/issues).
 
 ## How to obtain CWS data?
 To crowdsource CWS data different data providers with application programming interfaces (API) exist, each with advantages and disadvantages, e.g.:
-- <a href="https://www.netatmo.com/">Netatmo</a>: <a href="https://dev.netatmo.com/">Netatmo Connect API</a>
-- <a href="https://synopticdata.com/">Synoptic</a>: <a href="https://developers.synopticdata.com/mesonet/">Mesonet API</a>
-- <a href="https://www.wow.metoffice.gov.uk/">Weather Observations Website (WOW)</a>: <a href="https://mowowprod.portal.azure-api.net/">WOW API</a>
+- [Netatmo](https://www.netatmo.com/): [Netatmo Connect API](https://dev.netatmo.com/)
+- [Synoptic](https://synopticdata.com/): [Mesonet API](https://developers.synopticdata.com/mesonet/)
+- [Weather Observations Website (WOW)](https://www.wow.metoffice.gov.uk/): [WOW API](https://mowowprod.portal.azure-api.net/)
 
 ## Reference
 Please reference the following open-access journal article when using CrowdQC+:
 
-Fenner, D., Bechtel, B., Demuzere, M., Kittner, J. and Meier, F.: CrowdQC+ – A quality-control for crowdsourced air-temperature observations enabling world-wide urban climate applications. Frontiers in Environmental Science.
+Fenner, D., Bechtel, B., Demuzere, M., Kittner, J. and Meier, F. (2021): CrowdQC+ – A quality-control for crowdsourced air-temperature observations enabling world-wide urban climate applications. Frontiers in Environmental Science 9: 720747. DOI: [10.3389/fenvs.2021.720747](https://doi.org/10.3389/fenvs.2021.720747).
 
 ## Licence
-CrowdQC+ is distributed under the <a href="http://www.gnu.org/licenses/gpl-3.0.en.html">GNU General Public License v3</a>.
+CrowdQC+ is distributed under the [GNU General Public License v3](http://www.gnu.org/licenses/gpl-3.0.en.html).