cache get_robotstxt()

ropensci · Mar 22, 2016 · 90ad735 · 90ad735
1 parent aed88fa
commit 90ad735
Showing 1 changed file with 11 additions and 1 deletion.
diff --git a/R/parse_robotstxt.R b/R/parse_robotstxt.R
@@ -1,11 +1,20 @@
+#' get_robotstxt() cache
+rt_cache <- new.env(parent=emptyenv())
+
 #' downloading robots.txt file
 #' @param domain domain from which to download robots.txt file
 #' @export
 get_robotstxt <- function(domain, warn=TRUE){
-  request <- httr::GET(paste0(domain, "/robots.txt"))
+  # get data from cache or do download
+  if( is.null(rt_cache[[domain]]) ){
+    request <- httr::GET(paste0(domain, "/robots.txt"))
+  }else{
+    request <- rt_cache[[domain]]
+  }
   # ok
   if( request$status < 400 ){
     rtxt <- httr::content(request,  encoding="UTF-8", as="text")
+    rt_cache[[domain]] <- request
   }
   # not found
   if( request$status == 404 ){
@@ -17,6 +26,7 @@ get_robotstxt <- function(domain, warn=TRUE){
       ))
     }
     rtxt <- ""
+    rt_cache[[domain]] <- request
   }
   # not ok
   if( !(request$status == 404 | request$status < 400) ){