Handle download errors in getDividends & getSplits

We fixed this issue in getSymbols() in #307. Basically, the curl
connection and/or download may fail for one symbol, and we don't want
to throw an error and stop processing all subsequent symbols.

Create a new retry.yahoo() function that throws the warning about the
first download failing, then reconstructs the URL and tries the download
again. It throws an error if the second download fails, like it did for
getSymbols().

Fixes #314.

R/getDividends.R

@@ -1,6 +1,7 @@
 `getDividends` <-
 function(Symbol,from='1970-01-01',to=Sys.Date(),env=parent.frame(),src='yahoo',
-         auto.assign=FALSE,auto.update=FALSE,verbose=FALSE,split.adjust=TRUE,...) {
+         auto.assign=FALSE,auto.update=FALSE,verbose=FALSE,split.adjust=TRUE,...,
+         curl.options=list()) {
 
   if(missing(env))
     env <- parent.frame(1)
@@ -17,7 +18,13 @@ function(Symbol,from='1970-01-01',to=Sys.Date(),env=parent.frame(),src='yahoo',
   yahoo.URL <- .yahooURL(Symbol.name, from.posix, to.posix,
                          "1d", "div", handle)
 
-  fr <- read.csv(curl::curl(yahoo.URL,handle=handle$ch))
+  conn <- curl::curl(yahoo.URL,handle=handle$ch)
+  fr <- try(read.csv(conn, as.is=TRUE), silent = TRUE)
+
+  if (inherits(fr, "try-error")) {
+    fr <- retry.yahoo(Symbol.name, from.posix, to.posix, "1d", "div", conn)
+  }
+
   fr <- xts(fr[,2],as.Date(fr[,1]))
   colnames(fr) <- paste(Symbol.name,'div',sep='.')
 

R/getSplits.R

@@ -1,6 +1,7 @@
 `getSplits` <-
 function(Symbol,from='1970-01-01',to=Sys.Date(),env=parent.frame(),src='yahoo',
-         auto.assign=FALSE,auto.update=FALSE,verbose=FALSE,...) {
+         auto.assign=FALSE,auto.update=FALSE,verbose=FALSE,...,
+         curl.options=list()) {
 
   # Function written by Joshua Ulrich, using
   # getSymbols.yahoo as a guide.
@@ -19,7 +20,12 @@ function(Symbol,from='1970-01-01',to=Sys.Date(),env=parent.frame(),src='yahoo',
   yahoo.URL <- .yahooURL(Symbol.name, from.posix, to.posix,
                          "1d", "split", handle)
 
-  fr <- read.csv(curl::curl(yahoo.URL, handle=handle$ch), as.is=TRUE)
+  conn <- curl::curl(yahoo.URL, handle=handle$ch)
+  fr <- try(read.csv(conn, as.is=TRUE), silent=TRUE)
+
+  if (inherits(fr, "try-error")) {
+    fr <- retry.yahoo(Symbol.name, from.posix, to.posix, "1d", "split", conn)
+  }
 
   if(NROW(fr)==0) {
     fr <- NA

R/getSymbols.R

@@ -338,23 +338,9 @@ function(Symbols,env,return.class='xts',index.class="Date",
        fr <- try(read.csv(conn, na.strings="null"), silent = TRUE)
 
        if (inherits(fr, "try-error")) {
-         # warn user about the failure
-         warning(Symbols.name, " download failed; trying again.",
-                 call. = FALSE, immediate. = TRUE)
-         # re-create handle
-         handle <- .getHandle(curl.options, force.new = TRUE)
-         # try again. must rebuild url with crumbs
-         yahoo.URL <- .yahooURL(Symbols.name, from.posix, to.posix,
-                                interval, "history", handle)
-         close(conn)
-         conn <- curl::curl(yahoo.URL, handle = handle$ch)
-         fr <- try(read.csv(conn, na.strings="null"), silent = TRUE)
-         # error if second attempt also failed
-         if (inherits(fr, "try-error")) {
-           close(conn)
-           stop(Symbols.name, " download failed after two attempts. Error",
-                " message:\n", attr(fr, "condition")$message, call. = FALSE)
-         }
+         fr <- retry.yahoo(Symbols.name, from.posix, to.posix, interval,
+                           "history", curl.options = curl.options,
+                           na.strings = NULL)
        }
 
        if(verbose) cat("done.\n")

R/tools.R

@@ -58,3 +58,38 @@ function(url, destfile, method, quiet = FALSE, mode = "w", cacheOK = TRUE,
     }
   }
 }
+
+retry.yahoo <-
+function(symbol,
+         from,
+         to,
+         interval,
+         type,
+         conn,
+         ...,
+         curl.options = list())
+{
+     warning(symbol, " download failed; trying again.",
+             call. = FALSE, immediate. = TRUE)
+
+     # re-create handle
+     handle <- .getHandle(curl.options, force.new = TRUE)
+
+     # try again. must rebuild url with crumbs
+     yahoo.URL <- .yahooURL(symbol, from, to, interval, type, handle)
+
+     close(conn)
+     conn <- curl::curl(yahoo.URL, handle = handle$ch)
+
+     fr <- try(read.csv(conn, ..., as.is = TRUE), silent = TRUE)
+
+     # error if second attempt also failed
+     if (inherits(fr, "try-error")) {
+         close(conn)
+         stop(symbol, " download failed after two attempts. Error",
+              " message:\n", attr(fr, "condition")$message, call. = FALSE)
+     }
+
+     # return data
+     return(fr)
+}

man/getDividends.Rd

@@ -14,7 +14,9 @@ getDividends(Symbol,
              auto.assign = FALSE, 
              auto.update = FALSE, 
              verbose = FALSE,
-             split.adjust = TRUE, ...)
+             split.adjust = TRUE,
+             ...,
+             curl.options = list())
 }
 %- maybe also 'usage' for other objects documented here.
 \arguments{
@@ -29,6 +31,7 @@ getDividends(Symbol,
   \item{split.adjust}{ adjust dividends for splits (\code{TRUE} by default
     because that's what Yahoo returns)}
   \item{\dots}{ currently unused }
+  \item{curl.options}{ options passed to \code{curl::curl} }
 }
 \details{
 Eventually destined to be a wrapper function along the lines

man/getSplits.Rd

@@ -13,7 +13,9 @@ getSplits(Symbol,
           src = "yahoo", 
           auto.assign = FALSE, 
           auto.update = FALSE, 
-          verbose = FALSE, ...)
+          verbose = FALSE,
+          ...,
+          curl.options = list())
 }
 %- maybe also 'usage' for other objects documented here.
 \arguments{
@@ -26,6 +28,7 @@ getSplits(Symbol,
   \item{auto.update}{ automatically add split to data object }
   \item{verbose}{ display status of retrieval }
   \item{\dots}{ currently unused }
+  \item{curl.options}{ options passed to \code{curl::curl} }
 }
 \details{
 Eventually destined to be a wrapper function along the lines