From 49829d7231e6d5408c572a0462a1a63f1801c82f Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Sun, 23 Sep 2018 10:37:59 -0600 Subject: [PATCH 1/8] Add Tiingo as getQuote() source Initial version. Waiting on feedback about correct batch size from Tiingo. See #247. --- R/getQuote.R | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/R/getQuote.R b/R/getQuote.R index 59005fde..479492e9 100644 --- a/R/getQuote.R +++ b/R/getQuote.R @@ -314,3 +314,41 @@ getQuote.av <- function(Symbols, api.key, ...) { rownames(output) <- output$Symbol return(output[, c("Trade Time", "Last", "Volume")]) } +getQuote.tiingo <- function(Symbols, api.key, ...) { + #docs: https://api.tiingo.com/docs/iex/realtime + importDefaults("getQuote.tiingo") + if (!hasArg("api.key")) { + stop("getQuote.tiingo: An API key is required (api.key). Registration at https://api.tiingo.com/.", call. = FALSE) + } + URL <- paste0("https://api.tiingo.com/iex/?token=", api.key, "&tickers=") + batch.size = 100L + result <- NULL + for (i in seq(1, length(Symbols), batch.size)) { + if (i > 1L) { + Sys.sleep(0.25) + cat("getQuote.tiingo downloading batch", i, ":", i + batch.size - 1L, "\n") + } + batchSymbols <- Symbols[i:min(nSymbols, i + batch.size - 1L)] + batchURL <- paste0(URL, paste(batchSymbols, collapse = ",")) + batch.result <- jsonlite::fromJSON(batchURL) + #set column data types for each batch so we dont get issues with rbind + for (cn in colnames(batch.result)) { + if (grepl("timestamp", cn, ignore.case = T)) { + batch.result[, cn] <- as.POSIXct(batch.result[, cn]) + } + else if (!(cn == "ticker")) { + batch.result[, cn] <- as.numeric(batch.result[, cn]) + } + } + result <- rbind(result, batch.result) + } + colnames(result) <- gsub("(^[[:alpha:]])", "\\U\\1", colnames(result), perl = TRUE) + result$`Trade Time` <- result$LastsaleTimeStamp + + # merge join to produce empty rows for missing results from AV + # so that return value has the same number of rows and order as the input + output <- merge(data.frame(Ticker = Symbols), result, by = "Ticker", all.x = TRUE) + rownames(output) <- output$Ticker + std.cols <- c("Trade Time", "Open", "High", "Low", "Last", "Volume") + return(output[, c(std.cols, setdiff(colnames(output), c(std.cols, "Ticker")))]) +} From 333fbc4e2882542597e4ce5cdb46a4b449b1514d Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Mon, 15 Oct 2018 17:33:43 -0600 Subject: [PATCH 2/8] Import all symbols if Symbols = NULL Add support for getting all symbols with NULL symbols parameter. See #247. --- R/getQuote.R | 58 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/R/getQuote.R b/R/getQuote.R index 479492e9..d1e440d5 100644 --- a/R/getQuote.R +++ b/R/getQuote.R @@ -314,41 +314,49 @@ getQuote.av <- function(Symbols, api.key, ...) { rownames(output) <- output$Symbol return(output[, c("Trade Time", "Last", "Volume")]) } -getQuote.tiingo <- function(Symbols, api.key, ...) { + +`getQuote.tiingo` <- function(Symbols, api.key, ...) { #docs: https://api.tiingo.com/docs/iex/realtime + #NULL Symbols will retrive quotes for all symbols importDefaults("getQuote.tiingo") if (!hasArg("api.key")) { stop("getQuote.tiingo: An API key is required (api.key). Registration at https://api.tiingo.com/.", call. = FALSE) } - URL <- paste0("https://api.tiingo.com/iex/?token=", api.key, "&tickers=") - batch.size = 100L - result <- NULL - for (i in seq(1, length(Symbols), batch.size)) { - if (i > 1L) { - Sys.sleep(0.25) - cat("getQuote.tiingo downloading batch", i, ":", i + batch.size - 1L, "\n") - } - batchSymbols <- Symbols[i:min(nSymbols, i + batch.size - 1L)] - batchURL <- paste0(URL, paste(batchSymbols, collapse = ",")) - batch.result <- jsonlite::fromJSON(batchURL) - #set column data types for each batch so we dont get issues with rbind - for (cn in colnames(batch.result)) { - if (grepl("timestamp", cn, ignore.case = T)) { - batch.result[, cn] <- as.POSIXct(batch.result[, cn]) - } - else if (!(cn == "ticker")) { - batch.result[, cn] <- as.numeric(batch.result[, cn]) + + if(is.null(Symbols)) { + URL <- paste0("https://api.tiingo.com/iex/?token=", api.key) + r <- jsonlite::fromJSON(URL) + } else { + URL <- paste0("https://api.tiingo.com/iex/?token=", api.key, "&tickers=") + batch.size = 100L + r <- NULL + for (i in seq(1, length(Symbols), batch.size)) { + if (i > 1L) { + Sys.sleep(0.25) + cat("getQuote.tiingo downloading batch", i, ":", i + batch.size - 1L, "\n") } + batchSymbols <- Symbols[i:min(Symbols, i + batch.size - 1L)] + batchURL <- paste0(URL, paste(batchSymbols, collapse = ",")) + batch.result <- jsonlite::fromJSON(batchURL) + r <- rbind(r, batch.result) + } + } + #set column data types for each batch so we dont get issues with rbind + for (cn in colnames(r)) { + if (grepl("timestamp", cn, ignore.case = T)) { + r[, cn] <- as.POSIXct(r[, cn]) + } + else if (!(cn == "ticker")) { + r[, cn] <- as.numeric(r[, cn]) } - result <- rbind(result, batch.result) } - colnames(result) <- gsub("(^[[:alpha:]])", "\\U\\1", colnames(result), perl = TRUE) - result$`Trade Time` <- result$LastsaleTimeStamp + colnames(r) <- gsub("(^[[:alpha:]])", "\\U\\1", colnames(r), perl = TRUE) + r$`Trade Time` <- r$LastsaleTimeStamp # merge join to produce empty rows for missing results from AV # so that return value has the same number of rows and order as the input - output <- merge(data.frame(Ticker = Symbols), result, by = "Ticker", all.x = TRUE) - rownames(output) <- output$Ticker + if(!is.null(Symbols)) r <- merge(data.frame(Ticker = Symbols), r, by = "Ticker", all.x = TRUE) + rownames(r) <- r$Ticker std.cols <- c("Trade Time", "Open", "High", "Low", "Last", "Volume") - return(output[, c(std.cols, setdiff(colnames(output), c(std.cols, "Ticker")))]) + return(r[, c(std.cols, setdiff(colnames(r), c(std.cols, "Ticker")))]) } From 467d76b39eb2b5f8473dac4bb50296f875e7730e Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Tue, 16 Oct 2018 07:28:33 -0600 Subject: [PATCH 3/8] Move column fixups back into batch loop See #247. --- R/getQuote.R | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/R/getQuote.R b/R/getQuote.R index d1e440d5..086c0e27 100644 --- a/R/getQuote.R +++ b/R/getQuote.R @@ -323,10 +323,7 @@ getQuote.av <- function(Symbols, api.key, ...) { stop("getQuote.tiingo: An API key is required (api.key). Registration at https://api.tiingo.com/.", call. = FALSE) } - if(is.null(Symbols)) { - URL <- paste0("https://api.tiingo.com/iex/?token=", api.key) - r <- jsonlite::fromJSON(URL) - } else { + if(!is.null(Symbols)) { URL <- paste0("https://api.tiingo.com/iex/?token=", api.key, "&tickers=") batch.size = 100L r <- NULL @@ -338,18 +335,33 @@ getQuote.av <- function(Symbols, api.key, ...) { batchSymbols <- Symbols[i:min(Symbols, i + batch.size - 1L)] batchURL <- paste0(URL, paste(batchSymbols, collapse = ",")) batch.result <- jsonlite::fromJSON(batchURL) + print(tail(batch.result)) + #do type conversions for each batch so we deont get issues with rbind + for (cn in colnames(batch.result)) { + if (grepl("timestamp", cn, ignore.case = T)) { + batch.result[, cn] <- as.POSIXct(batch.result[, cn]) + } + else if (!(cn == "ticker")) { + batch.result[, cn] <- as.numeric(batch.result[, cn]) + } + } r <- rbind(r, batch.result) } - } - #set column data types for each batch so we dont get issues with rbind - for (cn in colnames(r)) { - if (grepl("timestamp", cn, ignore.case = T)) { - r[, cn] <- as.POSIXct(r[, cn]) - } - else if (!(cn == "ticker")) { - r[, cn] <- as.numeric(r[, cn]) + } else { + URL <- paste0("https://api.tiingo.com/iex/?token=", api.key) + batch.result <- jsonlite::fromJSON(URL) + print(tail(batch.result)) + for (cn in colnames(batch.result)) { + if (grepl("timestamp", cn, ignore.case = T)) { + batch.result[, cn] <- as.POSIXct(batch.result[, cn]) + } + else if (!(cn == "ticker")) { + batch.result[, cn] <- as.numeric(batch.result[, cn]) + } } + r <- batch.result } + colnames(r) <- gsub("(^[[:alpha:]])", "\\U\\1", colnames(r), perl = TRUE) r$`Trade Time` <- r$LastsaleTimeStamp From 228d04109eae26c08795d82b2b10240108e42423 Mon Sep 17 00:00:00 2001 From: Ethan Smith <24379655+ethanbsmith@users.noreply.github.com> Date: Fri, 9 Nov 2018 09:24:17 -0700 Subject: [PATCH 4/8] Remove duplicate code for (non)null Symbols See #247. --- R/getQuote.R | 50 +++++++++++++++++++++++--------------------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/R/getQuote.R b/R/getQuote.R index 086c0e27..ea18a073 100644 --- a/R/getQuote.R +++ b/R/getQuote.R @@ -323,34 +323,30 @@ getQuote.av <- function(Symbols, api.key, ...) { stop("getQuote.tiingo: An API key is required (api.key). Registration at https://api.tiingo.com/.", call. = FALSE) } - if(!is.null(Symbols)) { - URL <- paste0("https://api.tiingo.com/iex/?token=", api.key, "&tickers=") + base.url <- paste0("https://api.tiingo.com/iex/?token=", api.key) + r <- NULL + if (is.null(Symbols)) { + batch.size = 1L + batch.length =1L + } else { batch.size = 100L - r <- NULL - for (i in seq(1, length(Symbols), batch.size)) { - if (i > 1L) { - Sys.sleep(0.25) - cat("getQuote.tiingo downloading batch", i, ":", i + batch.size - 1L, "\n") - } - batchSymbols <- Symbols[i:min(Symbols, i + batch.size - 1L)] - batchURL <- paste0(URL, paste(batchSymbols, collapse = ",")) - batch.result <- jsonlite::fromJSON(batchURL) - print(tail(batch.result)) - #do type conversions for each batch so we deont get issues with rbind - for (cn in colnames(batch.result)) { - if (grepl("timestamp", cn, ignore.case = T)) { - batch.result[, cn] <- as.POSIXct(batch.result[, cn]) - } - else if (!(cn == "ticker")) { - batch.result[, cn] <- as.numeric(batch.result[, cn]) - } - } - r <- rbind(r, batch.result) + batch.length = length(Symbols) + } + + for (i in seq(1L, batch.length, batch.size)) { + if (i > 1L) { + Sys.sleep(0.25) + cat("getQuote.tiingo downloading batch", i, ":", i + batch.size - 1L, "\n") } - } else { - URL <- paste0("https://api.tiingo.com/iex/?token=", api.key) - batch.result <- jsonlite::fromJSON(URL) - print(tail(batch.result)) + + if (is.null(Symbols)) { + batch.url <- base.url + } else { + batch.url <- paste0(base.url, "&tickers=", paste(Symbols[i:min(Symbols, i + batch.size - 1L)], collapse = ",")) + } + + batch.result <- jsonlite::fromJSON(batch.url) + #do type conversions for each batch so we dont get issues with rbind for (cn in colnames(batch.result)) { if (grepl("timestamp", cn, ignore.case = T)) { batch.result[, cn] <- as.POSIXct(batch.result[, cn]) @@ -359,7 +355,7 @@ getQuote.av <- function(Symbols, api.key, ...) { batch.result[, cn] <- as.numeric(batch.result[, cn]) } } - r <- batch.result + r <- rbind(r, batch.result) } colnames(r) <- gsub("(^[[:alpha:]])", "\\U\\1", colnames(r), perl = TRUE) From e5d5facff7168cba7c7eab463aa433e54a326941 Mon Sep 17 00:00:00 2001 From: Joshua Ulrich Date: Sat, 10 Nov 2018 08:04:39 -0600 Subject: [PATCH 5/8] Indent to 2 spaces, correct typos --- R/getQuote.R | 86 ++++++++++++++++++++++++++-------------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/R/getQuote.R b/R/getQuote.R index ea18a073..3d0c5881 100644 --- a/R/getQuote.R +++ b/R/getQuote.R @@ -316,55 +316,55 @@ getQuote.av <- function(Symbols, api.key, ...) { } `getQuote.tiingo` <- function(Symbols, api.key, ...) { - #docs: https://api.tiingo.com/docs/iex/realtime - #NULL Symbols will retrive quotes for all symbols - importDefaults("getQuote.tiingo") - if (!hasArg("api.key")) { - stop("getQuote.tiingo: An API key is required (api.key). Registration at https://api.tiingo.com/.", call. = FALSE) + # docs: https://api.tiingo.com/docs/iex/realtime + # NULL Symbols will retrieve quotes for all symbols + importDefaults("getQuote.tiingo") + if(!hasArg("api.key")) { + stop("getQuote.tiingo: An API key is required (api.key). Registration at https://api.tiingo.com/.", call. = FALSE) + } + + base.url <- paste0("https://api.tiingo.com/iex/?token=", api.key) + r <- NULL + if(is.null(Symbols)) { + batch.size = 1L + batch.length = 1L + } else { + batch.size = 100L + batch.length = length(Symbols) + } + + for(i in seq(1L, batch.length, batch.size)) { + if(i > 1L) { + Sys.sleep(0.25) + cat("getQuote.tiingo downloading batch", i, ":", i + batch.size - 1L, "\n") } - base.url <- paste0("https://api.tiingo.com/iex/?token=", api.key) - r <- NULL - if (is.null(Symbols)) { - batch.size = 1L - batch.length =1L + if(is.null(Symbols)) { + batch.url <- base.url } else { - batch.size = 100L - batch.length = length(Symbols) + batch.url <- paste0(base.url, "&tickers=", paste(Symbols[i:min(Symbols, i + batch.size - 1L)], collapse = ",")) } - for (i in seq(1L, batch.length, batch.size)) { - if (i > 1L) { - Sys.sleep(0.25) - cat("getQuote.tiingo downloading batch", i, ":", i + batch.size - 1L, "\n") - } - - if (is.null(Symbols)) { - batch.url <- base.url - } else { - batch.url <- paste0(base.url, "&tickers=", paste(Symbols[i:min(Symbols, i + batch.size - 1L)], collapse = ",")) - } - - batch.result <- jsonlite::fromJSON(batch.url) - #do type conversions for each batch so we dont get issues with rbind - for (cn in colnames(batch.result)) { - if (grepl("timestamp", cn, ignore.case = T)) { - batch.result[, cn] <- as.POSIXct(batch.result[, cn]) - } - else if (!(cn == "ticker")) { - batch.result[, cn] <- as.numeric(batch.result[, cn]) - } - } - r <- rbind(r, batch.result) + batch.result <- jsonlite::fromJSON(batch.url) + # do type conversions for each batch so we don't get issues with rbind + for(cn in colnames(batch.result)) { + if(grepl("timestamp", cn, ignore.case = T)) { + batch.result[, cn] <- as.POSIXct(batch.result[, cn]) + } + else if(!(cn == "ticker")) { + batch.result[, cn] <- as.numeric(batch.result[, cn]) + } } + r <- rbind(r, batch.result) + } - colnames(r) <- gsub("(^[[:alpha:]])", "\\U\\1", colnames(r), perl = TRUE) - r$`Trade Time` <- r$LastsaleTimeStamp + colnames(r) <- gsub("(^[[:alpha:]])", "\\U\\1", colnames(r), perl = TRUE) + r$`Trade Time` <- r$LastsaleTimeStamp - # merge join to produce empty rows for missing results from AV - # so that return value has the same number of rows and order as the input - if(!is.null(Symbols)) r <- merge(data.frame(Ticker = Symbols), r, by = "Ticker", all.x = TRUE) - rownames(r) <- r$Ticker - std.cols <- c("Trade Time", "Open", "High", "Low", "Last", "Volume") - return(r[, c(std.cols, setdiff(colnames(r), c(std.cols, "Ticker")))]) + # merge join to produce empty rows for missing results from AV + # so that return value has the same number of rows and order as the input + if(!is.null(Symbols)) r <- merge(data.frame(Ticker = Symbols), r, by = "Ticker", all.x = TRUE) + rownames(r) <- r$Ticker + std.cols <- c("Trade Time", "Open", "High", "Low", "Last", "Volume") + return(r[, c(std.cols, setdiff(colnames(r), c(std.cols, "Ticker")))]) } From b7c85975e6bf149b7a21a571cd297583a16eb377 Mon Sep 17 00:00:00 2001 From: Joshua Ulrich Date: Sat, 10 Nov 2018 08:49:29 -0600 Subject: [PATCH 6/8] Fix column name typo and avoid $ subset The `$` data.frame subset method does partial column name matching, and returns NULL if the column name is not found. The `[` data.frame subset method will throw an error if the column name is not found. --- R/getQuote.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/getQuote.R b/R/getQuote.R index 3d0c5881..1a47f901 100644 --- a/R/getQuote.R +++ b/R/getQuote.R @@ -359,7 +359,7 @@ getQuote.av <- function(Symbols, api.key, ...) { } colnames(r) <- gsub("(^[[:alpha:]])", "\\U\\1", colnames(r), perl = TRUE) - r$`Trade Time` <- r$LastsaleTimeStamp + r[, "Trade Time"] <- r[, "LastSaleTimestamp"] # merge join to produce empty rows for missing results from AV # so that return value has the same number of rows and order as the input From 1e88e43a861828cda8d801c1b7c0e560c28e0b41 Mon Sep 17 00:00:00 2001 From: Joshua Ulrich Date: Sat, 10 Nov 2018 09:05:59 -0600 Subject: [PATCH 7/8] Fix URL if batch size is not multiple of 100 `min(Symbols, i + batch.size - 1L)` would always return the second argument because Symbols is a character vector. This should be `length(Symbols)`. Calculate the last batch observation index once at the top of the loop and use it in the printed message and when creating the URL. See #247. --- R/getQuote.R | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/R/getQuote.R b/R/getQuote.R index 1a47f901..45159607 100644 --- a/R/getQuote.R +++ b/R/getQuote.R @@ -334,15 +334,16 @@ getQuote.av <- function(Symbols, api.key, ...) { } for(i in seq(1L, batch.length, batch.size)) { + batch.end <- min(batch.length, i + batch.size - 1L) if(i > 1L) { Sys.sleep(0.25) - cat("getQuote.tiingo downloading batch", i, ":", i + batch.size - 1L, "\n") + cat("getQuote.tiingo downloading batch", i, ":", batch.end, "\n") } if(is.null(Symbols)) { batch.url <- base.url } else { - batch.url <- paste0(base.url, "&tickers=", paste(Symbols[i:min(Symbols, i + batch.size - 1L)], collapse = ",")) + batch.url <- paste0(base.url, "&tickers=", paste(Symbols[i:batch.end], collapse = ",")) } batch.result <- jsonlite::fromJSON(batch.url) From b96afcb4beead1af306a5daa86d7e2b90638815d Mon Sep 17 00:00:00 2001 From: Joshua Ulrich Date: Sat, 10 Nov 2018 09:10:38 -0600 Subject: [PATCH 8/8] Use consistent style/formatting --- R/getQuote.R | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/R/getQuote.R b/R/getQuote.R index 45159607..e00a40c6 100644 --- a/R/getQuote.R +++ b/R/getQuote.R @@ -320,17 +320,18 @@ getQuote.av <- function(Symbols, api.key, ...) { # NULL Symbols will retrieve quotes for all symbols importDefaults("getQuote.tiingo") if(!hasArg("api.key")) { - stop("getQuote.tiingo: An API key is required (api.key). Registration at https://api.tiingo.com/.", call. = FALSE) + stop("getQuote.tiingo: An API key is required (api.key). ", + "Registration at https://api.tiingo.com/.", call. = FALSE) } base.url <- paste0("https://api.tiingo.com/iex/?token=", api.key) r <- NULL if(is.null(Symbols)) { - batch.size = 1L - batch.length = 1L + batch.size <- 1L + batch.length <- 1L } else { - batch.size = 100L - batch.length = length(Symbols) + batch.size <- 100L + batch.length <- length(Symbols) } for(i in seq(1L, batch.length, batch.size)) { @@ -349,10 +350,10 @@ getQuote.av <- function(Symbols, api.key, ...) { batch.result <- jsonlite::fromJSON(batch.url) # do type conversions for each batch so we don't get issues with rbind for(cn in colnames(batch.result)) { - if(grepl("timestamp", cn, ignore.case = T)) { + if(grepl("timestamp", cn, ignore.case = TRUE)) { batch.result[, cn] <- as.POSIXct(batch.result[, cn]) } - else if(!(cn == "ticker")) { + else if(cn != "ticker") { batch.result[, cn] <- as.numeric(batch.result[, cn]) } }