From f445d9ea36b63a9dc389848dcf608da6339f3f63 Mon Sep 17 00:00:00 2001 From: ake <40662956+ake123@users.noreply.github.com> Date: Fri, 9 Jun 2023 11:42:56 +0300 Subject: [PATCH 1/7] Update get_eurostat_json.R --- R/get_eurostat_json.R | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/R/get_eurostat_json.R b/R/get_eurostat_json.R index c0797d93..7dcd78d4 100755 --- a/R/get_eurostat_json.R +++ b/R/get_eurostat_json.R @@ -116,7 +116,9 @@ get_eurostat_json <- function(id, # if (class(resp) == "try-error") { stop(paste("The requested url cannot be found within the get_eurostat_json function:", url)) } resp <- httr::RETRY("GET", url, terminate_on = c(404)) if (httr::http_error(resp)) { - stop(paste("The requested url cannot be found within the get_eurostat_json function:", url)) + stop(paste("The requested url cannot be found within the get_eurostat_json function: + Client Error - 100 No results found + Description - The requested resource is not available.", url)) } status <- httr::status_code(resp) @@ -135,12 +137,25 @@ get_eurostat_json <- function(id, jdat <- jsonlite::fromJSON(url) } else if (status == 400) { stop( - "Failure to get data. Probably invalid dataset id. Status code: ", + "Failure to get data. Probably invalid dataset id. + Client Errors - 100 No results found : The result from the query is empty. + - 140 Syntax error : The query is invalid. + - 150 Semantic error : + Description - The request is syntactically correct + but fails a semantic validation, business rules. + Status code: ", status, msg ) } else if (status == 500) { - stop("Failure to get data. Probably filters did not return any data - or data exceeded query size limitation. Status code: ", status, msg) + stop("Failure to get data. Probably filters did not return any data + or data exceeded query size limitation. + Fault Code - + Server Errors - 500 Internal Server error + Description - The webservice should return this error code when none of + the other error codes better describes the reason for the failure of + the service to provide a meaningful response.Also used in case of + request not respecting the xsd definition. + Status code: ", status, msg) } else if (status == 416) { stop( "Too many categories have been requested. Maximum is 50.", From 1adff6f30116b8d186a5db78cc6d118e24464bb6 Mon Sep 17 00:00:00 2001 From: ake <40662956+ake123@users.noreply.github.com> Date: Mon, 12 Jun 2023 11:47:39 +0300 Subject: [PATCH 2/7] Update get_eurostat_json.R Reading error message (http status, id and label) from the response and return the status, id and label that were saved in the memory. --- R/get_eurostat_json.R | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/R/get_eurostat_json.R b/R/get_eurostat_json.R index 7dcd78d4..3d697a25 100755 --- a/R/get_eurostat_json.R +++ b/R/get_eurostat_json.R @@ -116,9 +116,9 @@ get_eurostat_json <- function(id, # if (class(resp) == "try-error") { stop(paste("The requested url cannot be found within the get_eurostat_json function:", url)) } resp <- httr::RETRY("GET", url, terminate_on = c(404)) if (httr::http_error(resp)) { - stop(paste("The requested url cannot be found within the get_eurostat_json function: - Client Error - 100 No results found - Description - The requested resource is not available.", url)) + stop("Status : ",httr::content(resp)$error[[1]]$status, + ", error id : ",httr::content(resp)$error[[1]]$id, + ", label : ", httr::content(resp)$error[[1]]$label) } status <- httr::status_code(resp) @@ -136,26 +136,13 @@ get_eurostat_json <- function(id, if (status == 200) { jdat <- jsonlite::fromJSON(url) } else if (status == 400) { - stop( - "Failure to get data. Probably invalid dataset id. - Client Errors - 100 No results found : The result from the query is empty. - - 140 Syntax error : The query is invalid. - - 150 Semantic error : - Description - The request is syntactically correct - but fails a semantic validation, business rules. - Status code: ", - status, msg - ) + stop("Status : ",httr::content(resp)$error[[1]]$status, + ", error id : ",httr::content(resp)$error[[1]]$id, + ", label : ", httr::content(resp)$error[[1]]$label) } else if (status == 500) { - stop("Failure to get data. Probably filters did not return any data - or data exceeded query size limitation. - Fault Code - - Server Errors - 500 Internal Server error - Description - The webservice should return this error code when none of - the other error codes better describes the reason for the failure of - the service to provide a meaningful response.Also used in case of - request not respecting the xsd definition. - Status code: ", status, msg) + stop("Status : ",httr::content(resp)$error[[1]]$status, + ", error id : ",httr::content(resp)$error[[1]]$id, + ", label : ", httr::content(resp)$error[[1]]$label) } else if (status == 416) { stop( "Too many categories have been requested. Maximum is 50.", From 4f5a54eea6bf962c461257881e7829cceeb48c01 Mon Sep 17 00:00:00 2001 From: ake <40662956+ake123@users.noreply.github.com> Date: Tue, 13 Jun 2023 16:53:46 +0300 Subject: [PATCH 3/7] Update get_eurostat_json.R --- R/get_eurostat_json.R | 66 +++++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/R/get_eurostat_json.R b/R/get_eurostat_json.R index 3d697a25..5ab74971 100755 --- a/R/get_eurostat_json.R +++ b/R/get_eurostat_json.R @@ -106,7 +106,13 @@ get_eurostat_json <- function(id, message("You have no access to ec.europe.eu. Please check your connection and/or review your proxy settings") } else { - + msg <- ". Some datasets are not accessible via the eurostat + interface. You can try to search the data manually from the comext + database at http://epp.eurostat.ec.europa.eu/newxtweb/ or bulk + download facility at + http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing + or annual Excel files + http://ec.europa.eu/eurostat/web/prodcom/data/excel-files-nace-rev.2" # get response # url <- try(eurostat_json_url(id = id, filters = filters, lang = lang)) # if (class(url) == "try-error") { stop(paste("The requested data set cannot be found with the following specifications to get_eurostat_json function: ", "id: ", id, "/ filters: ", filters, "/ lang: ", lang)) } @@ -115,42 +121,60 @@ get_eurostat_json <- function(id, # resp <- try(httr::GET(url, ...)) # if (class(resp) == "try-error") { stop(paste("The requested url cannot be found within the get_eurostat_json function:", url)) } resp <- httr::RETRY("GET", url, terminate_on = c(404)) + #print(httr::http_error(resp)) + print(httr::headers(resp)) + #print(httr::text_content(resp)) if (httr::http_error(resp)) { - stop("Status : ",httr::content(resp)$error[[1]]$status, - ", error id : ",httr::content(resp)$error[[1]]$id, - ", label : ", httr::content(resp)$error[[1]]$label) + # stop(paste("The requested url cannot be found within the get_eurostat_json function: + # Client Error - 100 No results found + # Description - The requested resource is not available.", url)) + # + stop(paste("Status : ",httr::content(resp)$error[[1]]$status, + ", error id : ",httr::content(resp)$error[[1]]$id,"No results found", + ", label : ", httr::content(resp)$error[[1]]$label),"\n", + msg) + } } status <- httr::status_code(resp) - + # check status and get json - msg <- ". Some datasets are not accessible via the eurostat - interface. You can try to search the data manually from the comext - database at http://epp.eurostat.ec.europa.eu/newxtweb/ or bulk - download facility at - http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing - or annual Excel files - http://ec.europa.eu/eurostat/web/prodcom/data/excel-files-nace-rev.2" + # msg <- ". Some datasets are not accessible via the eurostat + # interface. You can try to search the data manually from the comext + # database at http://epp.eurostat.ec.europa.eu/newxtweb/ or bulk + # download facility at + # http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing + # or annual Excel files + # http://ec.europa.eu/eurostat/web/prodcom/data/excel-files-nace-rev.2" if (status == 200) { jdat <- jsonlite::fromJSON(url) } else if (status == 400) { - stop("Status : ",httr::content(resp)$error[[1]]$status, - ", error id : ",httr::content(resp)$error[[1]]$id, - ", label : ", httr::content(resp)$error[[1]]$label) + + stop(paste(paste("Status : ",httr::content(resp)$error[[1]]$status,"Bad request", + if(httr::content(resp)$error[[1]]$id ==100){paste(", error id : ",httr::content(resp)$error[[1]]$id , "No result found")} + else if(httr::content(resp)$error[[1]]$id ==140){paste(", error id : ",httr::content(resp)$error[[1]]$id , "Syntax error")} + else if(httr::content(resp)$error[[1]]$id ==150){paste(", error id : ",httr::content(resp)$error[[1]]$id , "Semantic error")}, + #", error id : ",httr::content(resp)$error[[1]]$id, + ", label : ", httr::content(resp)$error[[1]]$label,"\n", + msg))) } else if (status == 500) { - stop("Status : ",httr::content(resp)$error[[1]]$status, - ", error id : ",httr::content(resp)$error[[1]]$id, - ", label : ", httr::content(resp)$error[[1]]$label) + + stop(paste("Status : ",httr::content(resp)$error[[1]]$status,"Internal Server error", + ", error id : ",httr::content(resp)$error[[1]]$id," Internal Server error", + ", label : ", httr::content(resp)$error[[1]]$label),"\n", + msg) } else if (status == 416) { stop( "Too many categories have been requested. Maximum is 50.", - status, msg + status, + msg ) } else { - stop("Failure to get data. Status code: ", status, msg) + stop(paste("Failure to get data. Status code: ",msg)) } + # get json data # dims <- jdat[[1]]$dimension # Was called like this in API v1.1 @@ -190,7 +214,7 @@ get_eurostat_json <- function(id, tibble::as_tibble(dat) } -} + From fee43b94561211d7c5abfa0e1a045636c3f346d7 Mon Sep 17 00:00:00 2001 From: ake <40662956+ake123@users.noreply.github.com> Date: Tue, 13 Jun 2023 16:54:54 +0300 Subject: [PATCH 4/7] Update get_eurostat_json.R --- R/get_eurostat_json.R | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/R/get_eurostat_json.R b/R/get_eurostat_json.R index 5ab74971..9f6de6e8 100755 --- a/R/get_eurostat_json.R +++ b/R/get_eurostat_json.R @@ -121,9 +121,7 @@ get_eurostat_json <- function(id, # resp <- try(httr::GET(url, ...)) # if (class(resp) == "try-error") { stop(paste("The requested url cannot be found within the get_eurostat_json function:", url)) } resp <- httr::RETRY("GET", url, terminate_on = c(404)) - #print(httr::http_error(resp)) - print(httr::headers(resp)) - #print(httr::text_content(resp)) + if (httr::http_error(resp)) { # stop(paste("The requested url cannot be found within the get_eurostat_json function: # Client Error - 100 No results found From a86d575bd2482ade8cb09b7e1a51ef3102247660 Mon Sep 17 00:00:00 2001 From: ake <40662956+ake123@users.noreply.github.com> Date: Wed, 14 Jun 2023 16:10:40 +0300 Subject: [PATCH 5/7] Update get_eurostat_json.R --- R/get_eurostat_json.R | 44 ++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/R/get_eurostat_json.R b/R/get_eurostat_json.R index 9f6de6e8..42d1260c 100755 --- a/R/get_eurostat_json.R +++ b/R/get_eurostat_json.R @@ -107,12 +107,9 @@ get_eurostat_json <- function(id, Please check your connection and/or review your proxy settings") } else { msg <- ". Some datasets are not accessible via the eurostat - interface. You can try to search the data manually from the comext - database at http://epp.eurostat.ec.europa.eu/newxtweb/ or bulk - download facility at - http://ec.europa.eu/eurostat/estat-navtree-portlet-prod/BulkDownloadListing - or annual Excel files - http://ec.europa.eu/eurostat/web/prodcom/data/excel-files-nace-rev.2" + interface. You can try to search the data manually from the comext + database athttps://ec.europa.eu/eurostat/comext/newxtweb/ or API data query at + https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+data+query" # get response # url <- try(eurostat_json_url(id = id, filters = filters, lang = lang)) # if (class(url) == "try-error") { stop(paste("The requested data set cannot be found with the following specifications to get_eurostat_json function: ", "id: ", id, "/ filters: ", filters, "/ lang: ", lang)) } @@ -122,19 +119,23 @@ get_eurostat_json <- function(id, # if (class(resp) == "try-error") { stop(paste("The requested url cannot be found within the get_eurostat_json function:", url)) } resp <- httr::RETRY("GET", url, terminate_on = c(404)) + status <- httr::status_code(resp) + id <- httr::content(resp)$error[[1]]$id + label_des <-httr::content(resp)$error[[1]]$label + if (httr::http_error(resp)) { # stop(paste("The requested url cannot be found within the get_eurostat_json function: # Client Error - 100 No results found # Description - The requested resource is not available.", url)) # - stop(paste("Status : ",httr::content(resp)$error[[1]]$status, - ", error id : ",httr::content(resp)$error[[1]]$id,"No results found", - ", label : ", httr::content(resp)$error[[1]]$label),"\n", + paste("Status : ",status, + ", error id : ",id,"No results found", + ", label : ", label_des,"\n", msg) } } - status <- httr::status_code(resp) + #status <- httr::status_code(resp) # check status and get json @@ -150,18 +151,17 @@ get_eurostat_json <- function(id, jdat <- jsonlite::fromJSON(url) } else if (status == 400) { - stop(paste(paste("Status : ",httr::content(resp)$error[[1]]$status,"Bad request", - if(httr::content(resp)$error[[1]]$id ==100){paste(", error id : ",httr::content(resp)$error[[1]]$id , "No result found")} - else if(httr::content(resp)$error[[1]]$id ==140){paste(", error id : ",httr::content(resp)$error[[1]]$id , "Syntax error")} - else if(httr::content(resp)$error[[1]]$id ==150){paste(", error id : ",httr::content(resp)$error[[1]]$id , "Semantic error")}, - #", error id : ",httr::content(resp)$error[[1]]$id, - ", label : ", httr::content(resp)$error[[1]]$label,"\n", - msg))) + stop(paste("Status : ",status,"Bad request", + if(id ==100){paste(", error id : ",id , "No result found")} + else if(id ==140){paste(", error id : ",id , "Syntax error")} + else if(id ==150){paste(", error id : ",id , "Semantic error")}, + ", label : ", label_des,"\n", + msg)) } else if (status == 500) { - stop(paste("Status : ",httr::content(resp)$error[[1]]$status,"Internal Server error", - ", error id : ",httr::content(resp)$error[[1]]$id," Internal Server error", - ", label : ", httr::content(resp)$error[[1]]$label),"\n", + stop(paste("Status : ",status,"Internal Server error", + ", error id : ",id," Internal Server error", + ", label : ", label_des),"\n", msg) } else if (status == 416) { stop( @@ -170,7 +170,9 @@ get_eurostat_json <- function(id, msg ) } else { - stop(paste("Failure to get data. Status code: ",msg)) + stop(paste("Failure to get data. Status code: ", + status, + msg)) } From 6614b1267e6b2b788a85c1024502f4a15121ba1b Mon Sep 17 00:00:00 2001 From: ake123 Date: Fri, 16 Jun 2023 16:05:47 +0300 Subject: [PATCH 6/7] Update get_eurostat_json --- R/get_eurostat_json.R | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/R/get_eurostat_json.R b/R/get_eurostat_json.R index 42d1260c..36de1928 100755 --- a/R/get_eurostat_json.R +++ b/R/get_eurostat_json.R @@ -106,10 +106,7 @@ get_eurostat_json <- function(id, message("You have no access to ec.europe.eu. Please check your connection and/or review your proxy settings") } else { - msg <- ". Some datasets are not accessible via the eurostat - interface. You can try to search the data manually from the comext - database athttps://ec.europa.eu/eurostat/comext/newxtweb/ or API data query at - https://wikis.ec.europa.eu/display/EUROSTATHELP/API+Statistics+-+data+query" + msg <- " Some datasets are not accessible via the eurostat interface. You can try to search the data manually from the comext database at https://ec.europa.eu/eurostat/comext/newxtweb/ or explore data at https://ec.europa.eu/eurostat/web/main/data/database" # get response # url <- try(eurostat_json_url(id = id, filters = filters, lang = lang)) # if (class(url) == "try-error") { stop(paste("The requested data set cannot be found with the following specifications to get_eurostat_json function: ", "id: ", id, "/ filters: ", filters, "/ lang: ", lang)) } From b90e669289aa508b1116d5bef8383e3da923fdc9 Mon Sep 17 00:00:00 2001 From: ake123 Date: Mon, 19 Jun 2023 16:18:42 +0300 Subject: [PATCH 7/7] Update get_eurostat_json.R #262 --- R/get_eurostat_json.R | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/R/get_eurostat_json.R b/R/get_eurostat_json.R index 36de1928..5c08e830 100755 --- a/R/get_eurostat_json.R +++ b/R/get_eurostat_json.R @@ -116,10 +116,15 @@ get_eurostat_json <- function(id, # if (class(resp) == "try-error") { stop(paste("The requested url cannot be found within the get_eurostat_json function:", url)) } resp <- httr::RETRY("GET", url, terminate_on = c(404)) - status <- httr::status_code(resp) - id <- httr::content(resp)$error[[1]]$id - label_des <-httr::content(resp)$error[[1]]$label - + #status <- httr::status_code(resp) + #id <- httr::content(resp)$error[[1]]$id + #label_des <-httr::content(resp)$error[[1]]$label + result <- httr::content(resp)$error + json_data_frame <- as.data.frame(result) + status <- json_data_frame$status + id <- json_data_frame$id + label <- json_data_frame$label + #print(json_data_frame) if (httr::http_error(resp)) { # stop(paste("The requested url cannot be found within the get_eurostat_json function: # Client Error - 100 No results found @@ -127,7 +132,7 @@ get_eurostat_json <- function(id, # paste("Status : ",status, ", error id : ",id,"No results found", - ", label : ", label_des,"\n", + ", label : ", label,"\n", msg) } } @@ -148,17 +153,21 @@ get_eurostat_json <- function(id, jdat <- jsonlite::fromJSON(url) } else if (status == 400) { - stop(paste("Status : ",status,"Bad request", - if(id ==100){paste(", error id : ",id , "No result found")} - else if(id ==140){paste(", error id : ",id , "Syntax error")} - else if(id ==150){paste(", error id : ",id , "Semantic error")}, - ", label : ", label_des,"\n", - msg)) + # stop(paste("Status : ",status,"Bad request", + # if(id ==100){paste(", error id : ",id , "No result found")} + # else if(id ==140){paste(", error id : ",id , "Syntax error")} + # else if(id ==150){paste(", error id : ",id , "Semantic error")}, + # ", label : ", label,"\n", + # msg)) + stop(paste("Status :",status,"Bad request", + if(100 %in% id){paste(" errod id: ", id , "No result found")} + else if(140 %in% id){paste("error id: ",id , "Syntax error",)} + else if(150 %in% id){paste("errod id: ",id , "Semantic error")},label)) } else if (status == 500) { stop(paste("Status : ",status,"Internal Server error", ", error id : ",id," Internal Server error", - ", label : ", label_des),"\n", + ", label : ", label),"\n", msg) } else if (status == 416) { stop( @@ -168,7 +177,9 @@ get_eurostat_json <- function(id, ) } else { stop(paste("Failure to get data. Status code: ", - status, + "Status : ",status, + ", error id : ",id,"No results found", + ", label : ", label,"\n", msg)) }