Merge pull request #221 from rOpenGov/devel

Devel

DESCRIPTION

@@ -1,8 +1,8 @@
 Package: eurostat
 Type: Package
 Title: Tools for Eurostat Open Data
-Date: 2021-06-20
-Version: 3.7.6
+Date: 2021-06-24
+Version: 3.7.7
 Encoding: UTF-8
 Authors@R: 
     c(person(given = "Leo",

R/deprecated_regions_functions.R

@@ -90,7 +90,6 @@ add_nuts_level <- function (dat, geo_labels = "geo") {
 #'    regions::validate_nuts_regions(dat)
 #'  }
 #'  
-#' @importFrom regions validate_nuts_regions
 #' @export
 
 harmonize_geo_code <- function (dat) {
@@ -135,7 +134,6 @@ harmonize_geo_code <- function (dat) {
 #'   )) 
 #'  
 #' recode_to_nuts_2016(test_regional_codes)
-#' 
 #' @importFrom regions recode_nuts
 #' @export
 
@@ -182,13 +180,12 @@ recode_to_nuts_2016 <- function (dat) {
 #'   )) 
 #'  
 #' recode_to_nuts_2013(test_regional_codes)
-#' 
 #' @importFrom regions recode_nuts
 #' @export
 
 recode_to_nuts_2013 <- function (dat) {
 
   warning("The 'recode_to_nuts_2013' function is deprecated. Use instead regions::recode_nuts(dat, nuts_year = 2013)")
 
-  regions::recode_nuts(dat, nuts_year = 2013)
+  recode_nuts(dat, nuts_year = 2013)
 }

R/eurostat-package.R

@@ -21,5 +21,8 @@
 #' See citation("eurostat")
 #' \url{https://ropengov.github.io/eurostat/}
 #' @examples library(eurostat)
+#' @section regions functions:
+#' For working with sub-national statistics the basic functions of the regions package 
+#' are imported \url{https://ropengov.github.io/regions/}.
 #' @keywords package
 NULL

R/get_eurostat_json.R

@@ -59,6 +59,11 @@ get_eurostat_json <- function(id, filters = NULL,
                               stringsAsFactors = FALSE,
                               ...){
 
+  ## Special products that must be built to matrix
+  ## User gets message to use iotables::iotables_download() and halt this operation.
+  user_want_stop <- special_id_values(id)
+  if ( user_want_stop ) return(NULL)
+
   # Check if you have access to ec.europe.eu. 
   if (!check_access_to_data()){
     message("You have no access to ec.europe.eu. 
@@ -97,6 +102,9 @@ get_eurostat_json <- function(id, filters = NULL,
     } else if (status == 500){
       stop("Failure to get data. Probably filters did not return any data 
          or data exceeded query size limitation. Status code: ", status, msg)
+    } else if (status == 416) {
+      stop ("Too many categories have been requested. Maximum is 50.", 
+            status, msg)
     } else {
       stop("Failure to get data. Status code: ", status, msg)
     }
@@ -119,7 +127,7 @@ get_eurostat_json <- function(id, filters = NULL,
         stop("Invalid type ", type)
       }
     })
-  
+
     variables <- expand.grid(dims_list, KEEP.OUT.ATTRS = FALSE, 
                            stringsAsFactors = stringsAsFactors)
 
@@ -130,6 +138,8 @@ get_eurostat_json <- function(id, filters = NULL,
     inds <- 1 + as.numeric(names(jdat$value)) # 0-indexed
     if (!length(vals) == length(inds)) {stop("Complex indexing not implemented.")}
     dat$values[inds] <- vals
+
+
 
     tibble::as_tibble(dat)
   }
@@ -164,8 +174,38 @@ eurostat_json_url <- function(id, filters, lang){
   url <- httr::build_url(url_list)
   url
 }
+# Internal function to give warning if symmetric input-output tables need to download into strict matirx formats.
+special_id_values <- function(id) {
+  siot_id_codes <- c("naio_10_cp1700", "naio_10_pyp1700",
+                     "naio_10_cp1750", "naio_10_pyp1750",
+                     "naio_10_cp15", "naio_10_cp16",
+                     "naio_10_cp1610","naio_10_pyp1610",
+                     "naio_10_cp1620", "naio_10_pyp1620",
+                     "naio_10_cp1630", "naio_10_pyp1630")
+  if (id %in% siot_id_codes ) {
+    message("The requested product id is a special input-output matrix.", 
+    "\nTo keep the matrix structure for further use, download it with iotables::iotables_download().\nThe iotables package is an extension for such cases to the eurostat package.")
+    answer <- readline (prompt = "Do you want to stop downloading now? [y/n] ")
+    if ( tolower(answer)=="y") TRUE else FALSE
+  } else {
+    # By default evaluates to FALSE and no interruption happens
+    FALSE
+  }
+}
 
+# Internal function to give warning if sub-national geo codes need validation
+is_regional_nuts_present <- function(geo) {
 
-
-
-
+  potentional_regional_codes <- unique (dat$geo)[nchar(unique (dat$geo))>2]
+
+  potentional_regional_codes <- potentional_regional_codes[! substr(potentional_regional_codes, 1,2) %in% c("EU", "EA")]
+
+  if( length(potentional_regional_codes)>0) {
+
+    types_found <- paste ( sort (unique(validate_geo_code( potentional_regional_codes, nuts_year = 2021))), collapse = ', ')
+    message("The following sub-national geographical codes present in the dataset:\n", types_found, 
+            "\nRegional and metropolitian area boundaries, codes and names are changing frequently.", 
+            "\nSee ?validate_geo_code, ?validate_nuts_regions and ?recode_nuts or the", 
+            "\n'Mapping Regional Data, Mapping Metadata Problems' vignette for a tutorial.")
+  }
+}

R/regional_coding_functions.R

@@ -0,0 +1,137 @@
+#' @title Recode Region Codes From Source To Target NUTS Typology
+#'
+#' Validate your geo codes, pair them with the appropriate standard
+#' typology, look up potential causes of invalidity in the EU correspondence
+#' tables, and look up the appropriate geographical codes in the other
+#' (target) typology.  For example, validate geo codes in the \code{'NUTS2016'}
+#' typology and translate them to the now obsolete the \code{'NUTS2010'} typology
+#' to join current data with historical data sets.
+#'  
+#' Imported from the \url{https://ropengov.github.io/regions/}{regions} package.
+#' 
+#' @param dat A data frame with a 3-5 character \code{geo_var} variable
+#' to be validated.
+#' @param geo_var Defaults to \code{"geo"}. The variable that contains
+#' the 3-5 character geo codes to be validated.
+#' @param nuts_year The year of the NUTS typology to use.
+#' You can select any valid
+#' NUTS definition, i.e. \code{1999}, \code{2003}, \code{2006},
+#' \code{2010}, \code{2013}, the currently used \code{2016} and the
+#' already announced and defined \code{2021}. Defaults to the current
+#' typology in force, which is \code{2016}.
+#' @importFrom regions recode_nuts
+#' @return The original data frame with a \code{'geo_var'} column is extended
+#' with a \code{'typology'} column that states in which typology is the \code{'geo_var'}
+#' a valid code.  For invalid codes, looks up potential reasons of invalidity
+#' and adds them to the \code{'typology_change'} column, and at last it
+#' adds a column of character vector containing the desired codes in the
+#' target typology, for example, in the NUTS2013 typology.
+#' @family regions functions
+#' @export
+#' @examples{
+#' foo <- data.frame (
+#'   geo  =  c("FR", "DEE32", "UKI3" ,
+#'             "HU12", "DED",
+#'             "FRK"),
+#'   values = runif(6, 0, 100 ),
+#'   stringsAsFactors = FALSE )
+#'
+#' recode_nuts(foo, nuts_year = 2013)
+#' }
+
+
+#' @title Validate Conformity With NUTS Geo Codes
+#' @description  Validate that \code{geo_var} is conforming with the \code{NUTS1},
+#' \code{NUTS2}, or \code{NUTS3} typologies.
+#' 
+#' @details While country codes are technically not part of the NUTS typologies,
+#' Eurostat de facto uses a \code{NUTS0} typology to identify countries.
+#' This de facto typology has three exception which are handled by the
+#' \link[regions]{validate_nuts_countries} function.
+#'
+#' NUTS typologies have different versions, therefore the conformity
+#' is validated with one specific versions, which can be any of these:
+#' \code{1999}, \code{2003}, \code{2006}, \code{2010},
+#'  \code{2013}, the currently used \code{2016} and the already
+#'  announced and defined \code{2021}.
+#'
+#' The NUTS typology was codified with the \code{NUTS2003}, and the
+#' pre-1999 NUTS typologies may confuse programmatic data processing,
+#' given that some  NUTS1 regions were identified with country codes
+#' in smaller countries that had no \code{NUTS1} divisions.
+#'
+#' Currently the \code{2016} is used by Eurostat, but many datasets
+#' still contain \code{2013} and sometimes earlier metadata.
+#'
+#' @param dat A data frame with a 3-5 character \code{geo_var}
+#' variable to be validated.
+#' @param geo_var Defaults to \code{"geo"}. The variable that contains
+#' the 3-5 character geo codes to be validated.
+#' @param nuts_year The year of the NUTS typology to use.
+#' Defaults to \code{2016}.  You can select any valid
+#' NUTS definition, i.e. \code{1999}, \code{2003}, \code{2006},
+#' \code{2010}, \code{2013}, the currently used \code{2016} and the
+#' already announced and defined \code{2021}.
+#' @importFrom regions validate_nuts_regions 
+#' @return Returns the original \code{dat} data frame with a column
+#' that specifies the comformity with the NUTS definition of the year
+#' \code{nuts_year}.
+#' @family regions functions
+#' @export
+#' @examples
+#' \donttest{
+#' my_reg_data <- data.frame (
+#'   geo = c("BE1", "HU102", "FR1",
+#'           "DED", "FR7", "TR", "DED2",
+#'           "EL", "XK", "GB"),
+#'   values = runif(10))
+#'
+#' validate_nuts_regions (my_reg_data)
+#'
+#' validate_nuts_regions (my_reg_data, nuts_year = 2013)
+#'
+#' validate_nuts_regions (my_reg_data, nuts_year = 2003)
+#' }
+
+
+
+#' @titel Validate Conformity with NUTS Geo Codes (vector)
+#'
+#' Validate that \code{geo} is conforming with the \code{NUTS1},
+#' \code{NUTS2}, or \code{NUTS3} typologies.
+#' While country codes are technically not part of the NUTS typologies,
+#' Eurostat de facto uses a \code{NUTS0} typology to identify countries.
+#' This de facto typology has three exception which are handled by the
+#' \link[regions]{validate_nuts_countries} function.
+#'
+#' NUTS typologies have different versions, therefore the conformity
+#' is validated with one specific versions, which can be any of these:
+#' \code{1999}, \code{2003}, \code{2006}, \code{2010},
+#'  \code{2013}, the currently used \code{2016} and the already
+#'  announced and defined \code{2021}.
+#'
+#' The NUTS typology was codified with the \code{NUTS2003}, and the
+#' pre-1999 NUTS typologies may confuse programmatic data processing,
+#' given that some  NUTS1 regions were identified with country codes
+#' in smaller countries that had no \code{NUTS1} divisions.
+#'
+#' Currently the \code{2016} is used by Eurostat, but many datasets
+#' still contain \code{2013} and sometimes earlier metadata.
+#' @param geo A vector of geographical code to validate.
+#' @param nuts_year A valid NUTS edition year.
+#' @importFrom regions validate_geo_code
+#' @family regions functions
+#' @return A character list with the valid typology, or 'invalid' in the cases
+#' when the geo coding is not valid.
+#' @export
+#' @examples
+#' \donttest{
+#' my_reg_data <- data.frame (
+#'   geo = c("BE1", "HU102", "FR1",
+#'           "DED", "FR7", "TR", "DED2",
+#'           "EL", "XK", "GB"),
+#'   values = runif(10))
+#'
+#' validate_geo_code(my_reg_data$geo)
+#' }
+

README.md

@@ -111,7 +111,7 @@ Eurostat. Searching for data is one way, if you know what to look for.
 <td style="text-align: left;">Air passenger transport by reporting country</td>
 <td style="text-align: left;">avia_paoc</td>
 <td style="text-align: left;">dataset</td>
-<td style="text-align: left;">16.06.2021</td>
+<td style="text-align: left;">21.06.2021</td>
 <td style="text-align: left;">26.05.2021</td>
 <td style="text-align: left;">1993</td>
 <td style="text-align: left;">2021Q1</td>
@@ -121,8 +121,8 @@ Eurostat. Searching for data is one way, if you know what to look for.
 <td style="text-align: left;">Air passenger transport by main airports in each reporting country</td>
 <td style="text-align: left;">avia_paoa</td>
 <td style="text-align: left;">dataset</td>
-<td style="text-align: left;">16.06.2021</td>
-<td style="text-align: left;">07.05.2021</td>
+<td style="text-align: left;">21.06.2021</td>
+<td style="text-align: left;">21.06.2021</td>
 <td style="text-align: left;">1993</td>
 <td style="text-align: left;">2021Q1</td>
 <td style="text-align: left;">NA</td>
@@ -131,7 +131,7 @@ Eurostat. Searching for data is one way, if you know what to look for.
 <td style="text-align: left;">Air passenger transport between reporting countries</td>
 <td style="text-align: left;">avia_paocc</td>
 <td style="text-align: left;">dataset</td>
-<td style="text-align: left;">16.06.2021</td>
+<td style="text-align: left;">21.06.2021</td>
 <td style="text-align: left;">26.05.2021</td>
 <td style="text-align: left;">1993</td>
 <td style="text-align: left;">2021Q1</td>
@@ -170,7 +170,7 @@ Lahti](https://github.com/antagomir), Przemyslaw Biecek, Markus Kainu
 and Janne Huovari. Retrieval and analysis of Eurostat open data with the
 eurostat package. [R Journal 9(1):385-392,
 2017](https://journal.r-project.org/archive/2017/RJ-2017-019/index.html).
-R package version 3.7.6. URL: <https://ropengov.github.io/eurostat/>
+R package version 3.7.7. URL: <https://ropengov.github.io/eurostat/>
 
 We are grateful to all
 [contributors](https://github.com/ropengov/eurostat/graphs/contributors),