Merge pull request #271 from rOpenGov/search_eurostat

Enhance search_eurostat(), TOC handling, documentation

.Rbuildignore

@@ -27,7 +27,6 @@ docs
 ^\.Rproj\.user$
 ^packrat/
 ^\.Rprofile$
-^NEWS.md$
 ^cran-comments\.md$
 ^data-raw$
 ^revdep$

DESCRIPTION

@@ -50,7 +50,8 @@ Imports:
     stringi,
     stringr,
     tibble,
-    tidyr (>= 1.0.0)
+    tidyr (>= 1.0.0),
+    xml2
 Suggests:
     giscoR,
     knitr,

NAMESPACE

@@ -10,6 +10,7 @@ export(eurotime2num)
 export(get_bibentry)
 export(get_eurostat)
 export(get_eurostat_dic)
+export(get_eurostat_folder)
 export(get_eurostat_geospatial)
 export(get_eurostat_json)
 export(get_eurostat_toc)
@@ -19,6 +20,7 @@ export(harmonize_geo_code)
 export(label_eurostat)
 export(label_eurostat_tables)
 export(label_eurostat_vars)
+export(label_eurostat_vars2)
 export(recode_nuts)
 export(recode_to_nuts_2013)
 export(recode_to_nuts_2016)
@@ -33,6 +35,7 @@ importFrom(classInt,classIntervals)
 importFrom(countrycode,countrycode)
 importFrom(curl,curl_download)
 importFrom(digest,digest)
+importFrom(dplyr,"%>%")
 importFrom(dplyr,case_when)
 importFrom(dplyr,coalesce)
 importFrom(dplyr,filter)
@@ -63,6 +66,7 @@ importFrom(rlang,sym)
 importFrom(stringi,stri_extract_first_regex)
 importFrom(stringi,stri_replace_all_fixed)
 importFrom(stringi,stri_replace_all_regex)
+importFrom(stringr,str_extract)
 importFrom(stringr,str_glue)
 importFrom(stringr,str_replace_all)
 importFrom(tibble,as_tibble)
@@ -71,5 +75,9 @@ importFrom(tidyr,pivot_longer)
 importFrom(tidyr,separate)
 importFrom(utils,download.file)
 importFrom(utils,hasName)
+importFrom(utils,menu)
 importFrom(utils,person)
 importFrom(utils,toBibtex)
+importFrom(xml2,read_xml)
+importFrom(xml2,xml_find_all)
+importFrom(xml2,xml_text)

NEWS.md

@@ -1,12 +1,33 @@
 # eurostat 4.0.0.9003
 
+## New features
+
+* `get_eurostat()` function now explicity accepts a 'lang' argument, for passing onwards to `get_eurostat_json()` and `label_eurostat()`
+* New user facing function: `get_eurostat_folder()` for downloading all datasets in a folder. The function is limited to downloading folders that contain at maximum 10 datasets. This function relies on new internal helper functions: `toc_count_whitespace()`, `toc_determine_hierarchy()`, `toc_count_children()` and `toc_list_children()`.
+
 ## Major updates
 
 * Rewritten caching functionalities, making it possible to cache filtered queries and rely on local caches if the user attempt to filter a complete dataset that has already been cached.
+* Column names in `.eurostatTOC` object (returned by `get_eurostat_toc()`) now use dots instead of spaces in the style of `base::make.names()`, e.g. turning `last update of data` to `last.update.of.data`
+* `.eurostatTOC` object includes a new hierarchy column that represents the position of each folder, dataset and table in the folder structure.
+* `search_eurostat()` includes the option to search Table of Content items by dataset codes in addition to titles. This makes it possible to make further queries from similar datasets (e.g. "nama_10_gdp", "nama_10r_2gdp", "nama_10r_3popgdp") that might have different titles.
+* `label_eurostat_tables()` has been rewritten to use the new SDMX API instead of `table_dic.dic` file in Eurostat Bulk Download Listing
+
+## Minor updates
+
+* Use more parameter inheritance in package function documentation to reduce discrepancies between different functions (DRY-principle)
+* Documentation more explicitly explains how to use filter parameters in `get_eurostat()` and `get_eurostat_json()` functions. The documentation now warns users about potential problems caused by `time` / `TIME_PERIOD` parameters when used to query datasets that contain quarterly data (issue #260)
+* As continuation of the update done in 3.7.14, started to use the new URL also for dictionary files in `get_eurostat_dic()` and `label_eurostat()` functions.
+
+## Deprecated and defunct
+
+* `grepEurostatTOC()` is completely marked as defunct and is enroute to being removed from the package as `search_eurostat()` is now the only way to fetch Eurostat TOC items and search (grep) them
+* `label_eurostat_vars()` has been marked as deprecated in favour of a new (temporary) function `label_eurostat_vars2()` which uses the new SDMX API to retrieve names for dataset columns. The old function will be completely removed after October 2023 when Eurostat Bulk Download Listing website is retired and `label_eurostat_vars2` will be renamed to `label_eurostat_vars()`. Function evolution is subject to ongoing Eurostat API developments.
 
 ## Bug fixes
 
-* `get_bibentry()` returns correct codes for titles and warns the user if some / all of the requested codes were not found in the TOC.
+* `get_bibentry()` returns correct codes for titles and warns the user if some / all of the requested codes were not found in the TOC. 
+* `get_bibentry()` uses the date field with the internal BibEntry format that can be easily translated to other formats (biblatex, built-in bibentry)
 
 # eurostat 4.0.0.9002
 

R/eurostat-defunct.R

@@ -0,0 +1,26 @@
+#' @title Defunct functions in eurostat
+#' 
+#' @description
+#' This list of defunct functions is maintained to document changes to eurostat functions in a
+#' transparent manner.
+#' 
+#' @param ... Generic representation of old arguments
+#' 
+#' @name eurostat-defunct
+#' @aliases eurostat-defunct
+#' 
+#' @details
+#' The following functions are defunct:
+#' 
+#' \itemize{
+#'  \item \code{\link{grepEurostatTOC}}: Use \code{search_eurostat} instead
+#' }
+#' 
+#' 
+NULL
+
+#' @rdname eurostat-defunct
+#' @export
+grepEurostatTOC <- function(...) {
+  .Defunct(new = "search_eurostat")
+}

R/eurostat-deprecated.R

@@ -0,0 +1,19 @@
+#' @name eurostat-deprecated
+#' @aliases eurostat-deprecated
+#' @title Deprecated functions in package eurostat
+#' 
+#' @description
+#' These functions are provided for compatibility with older versions of
+#' eurostat package only, and will be defunct at the next release.
+#' 
+#' @details
+#' The following functions are deprecated and will be made defunct; use the 
+#' replacement indicated below:
+#' 
+#' \itemize{
+#' 
+#'  \item{label_eurostat_vars: \code{\link{label_eurostat_vars2}}}
+#' 
+#' }
+#' 
+NULL