First stab at req_perform_sequential()

NAMESPACE

@@ -74,6 +74,7 @@ export(req_options)
 export(req_perform)
 export(req_perform_iteratively)
 export(req_perform_parallel)
+export(req_perform_sequential)
 export(req_perform_stream)
 export(req_progress)
 export(req_proxy)

NEWS.md

@@ -1,5 +1,11 @@
 # httr2 (development version)
 
+* New `req_perform_sequential()` which performs a known set of requests 
+  sequentially. It has an interface similar to `req_perform_parallel()` but
+  with no limitations, and the cost of being slower (#361).
+
+* All errors thrown by httr2 now inherit from the `httr2_error` class.
+
 * New `req_body_json_modify()` allows you to iteratively modify a JSON 
   body of a request.
 

R/multi-req.R

@@ -19,7 +19,9 @@
 #' * Consults the cache set by [req_cache()] before/after all requests.
 #'
 #' In general, where [req_perform()] might make multiple requests due to retries
-#' or OAuth failures, `req_perform_parallel()` will only make 1.
+#' or OAuth failures, `req_perform_parallel()` will only make 1. If any of
+#' these limitations are problematic, you may want to use
+#' [req_perform_sequential()] instead.
 #'
 #' @param reqs A list of [request]s.
 #' @param paths An optional list of paths, if you want to download the request

R/req-body.R

@@ -113,7 +113,7 @@ req_body_json_modify <- function(req, ...) {
     cli::cli_abort("Can only be used after {.fn req_body_json")
   }
 
-  req$body$data <- modifyList(req$body$data, list2(...))
+  req$body$data <- utils::modifyList(req$body$data, list2(...))
   req
 }
 

R/req-error.R

@@ -12,12 +12,12 @@
 #'
 #' * The HTTP request might fail, for example if the connection is dropped
 #'   or the server doesn't exist. This type of error will have class
-#'   `httr2_failure`.
+#'   `c("httr2_failure", "htt2_error")`.
 #'
 #' * The HTTP request might succeed, but return an HTTP status code that
 #'   represents a error, e.g. a `404 Not Found` if the specified resource is
 #'   not found. This type of error will have (e.g.) class
-#'   `c("httr2_http_404", "httr2_http")`.
+#'   `c("httr2_http_404", "httr2_http", "httr2_error")`.
 #'
 #' These error classes are designed to be used in conjunction with R's
 #' condition handling tools (<https://adv-r.hadley.nz/conditions.html>).

R/req-perform.R

@@ -110,7 +110,7 @@ req_perform <- function(
       error = function(err) {
         error_cnd(
           message = "Failed to perform HTTP request.",
-          class = "httr2_failure",
+          class = c("httr2_failure", "httr2_error"),
           parent = err,
           request = req,
           call = error_call,

R/resp-status.R

@@ -79,7 +79,7 @@ resp_abort <- function(resp, req, info = NULL, call = caller_env()) {
     c(message, resp_auth_message(resp), info),
     status = status,
     resp = resp,
-    class = c(glue("httr2_http_{status}"), "httr2_http"),
+    class = c(glue("httr2_http_{status}"), "httr2_http", "httr2_error"),
     request = req,
     call = call
   )

R/sequential.R

@@ -0,0 +1,94 @@
+#' Perform multiple requests in sequence
+#'
+#' Given a list of requests, this function performs each in turn, returning
+#' a list of responses. It's slower than [req_perform_parallel()] but
+#' has fewer limitations.
+#'
+#' @inheritParams req_perform_parallel
+#' @inheritParams req_perform_iteratively
+#' @param on_error What should happen if one of the requests throws an
+#'   HTTP error?
+#'
+#'   * `stop`, the default: stop iterating and throw an error
+#'   * `return`: stop iterating and return all the successful responses so far.
+#'   * `continue`: continue iterating, recording errors in the result.
+#' @export
+#' @return
+#' A list, the same length as `reqs`.
+#'
+#' If `on_error` is `"return"` and it errors on the ith request, the ith
+#' element of the result will be an error object, and the remaining elements
+#' will be `NULL`.
+#'
+#' If `on_error` is `"continue"`, it will be a mix of requests and errors.
+#' @examples
+#' # One use of req_perform_sequential() is if the API allows you to request
+#' # data for multiple objects, you want data for more objects than can fit
+#' # in one request.
+#' req <- request("https://api.restful-api.dev/objects")
+#'
+#' # Imagine we have 50 ids:
+#' ids <- sort(sample(100, 50))
+#'
+#' # But the API only allows us to request 10 at time. So we first use split
+#' # and some modulo arithmetic magic to generate chunks of length 10
+#' chunks <- unname(split(ids, (seq_along(ids) - 1) %/% 10))
+#'
+#' # Then we use lapply to generate one request for each chunk:
+#' reqs <- chunks %>% lapply(\(idx) req %>% req_url_query(id = idx, .multi = "comma"))
+#'
+#' # Then we can perform them all and get the results
+#' \dontrun{
+#' resps <- reqs %>% req_perform_sequential()
+#' resps_data(resps, \(resp) resp_body_json(resp))
+#' }
+req_perform_sequential <- function(reqs,
+                                   paths = NULL,
+                                   on_error = c("stop", "return", "continue"),
+                                   progress = TRUE) {
+  if (!is_bare_list(reqs)) {
+    stop_input_type(reqs, "a list")
+  }
+  if (!is.null(paths)) {
+    check_character(paths)
+    if (length(reqs) != length(paths)) {
+      cli::cli_abort("If supplied, {.arg paths} must be the same length as {.arg req}.")
+    }
+  }
+  on_error <- arg_match(on_error)
+  err_catch <- on_error != "stop"
+  err_return <- on_error == "return"
+
+  progress <- create_progress_bar(
+    total = length(reqs),
+    name = "Iterating",
+    config = progress
+  )
+
+  resps <- rep_along(reqs, list())
+
+  tryCatch({
+    for (i in seq_along(reqs)) {
+      check_request(reqs[[i]], arg = glue::glue("req[[{i}]]"))
+
+      if (err_catch) {
+        resps[[i]] <- tryCatch(
+          req_perform(reqs[[i]], path = paths[[i]]),
+          httr2_error = function(err) err
+        )
+      } else {
+        resps[[i]] <- req_perform(reqs[[i]], path = paths[[i]])
+      }
+      if (err_return && is_error(resps[[i]])) {
+        break
+      }
+      progress$update()
+    }
+  }, interrupt = function(cnd) {
+    resps <- resps[seq_len(i)]
+    cli::cli_alert_warning("Terminating iteration; returning {i} response{?s}.")
+  })
+  progress$done()
+
+  resps
+}

_pkgdown.yml

@@ -60,6 +60,7 @@ reference:
   contents:
   - req_perform_iteratively
   - req_perform_parallel
+  - req_perform_sequential
   - starts_with("iterate_")
   - starts_with("resps_")
 

tests/testthat/_snaps/sequential.md

@@ -0,0 +1,13 @@
+# checks its inputs
+
+    Code
+      req_perform_sequential(req)
+    Condition
+      Error in `req_perform_sequential()`:
+      ! `reqs` must be a list, not a <httr2_request> object.
+    Code
+      req_perform_sequential(list(req), letters)
+    Condition
+      Error in `req_perform_sequential()`:
+      ! If supplied, `paths` must be the same length as `req`.
+

tests/testthat/test-sequential.R

@@ -0,0 +1,57 @@
+test_that("checks its inputs", {
+  req <- request("http://example.com")
+  expect_snapshot(error = TRUE, {
+    req_perform_sequential(req)
+    req_perform_sequential(list(req), letters)
+  })
+})
+
+test_that("can download files", {
+  reqs <- list(request_test("/json"), request_test("/html"))
+  paths <- c(withr::local_tempfile(), withr::local_tempfile())
+  resps <- req_perform_sequential(reqs, paths)
+
+  expect_equal(resps[[1]]$body, new_path(paths[[1]]))
+  expect_equal(resps[[2]]$body, new_path(paths[[2]]))
+
+  # And check that something was downloaded
+  expect_gt(file.size(paths[[1]]), 0)
+  expect_gt(file.size(paths[[2]]), 0)
+})
+
+test_that("on_error = 'return' returns error", {
+  reqs <- list2(
+    request_test("/status/:status", status = 200),
+    request_test("/status/:status", status = 200),
+    request_test("/status/:status", status = 404),
+    request_test("/status/:status", status = 200)
+  )
+  out <- req_perform_sequential(reqs, on_error = "return")
+  expect_length(out, 4)
+  expect_s3_class(out[[3]], "httr2_http_404")
+  expect_equal(out[[4]], NULL)
+})
+
+
+test_that("on_error = 'continue' captures both error types", {
+  reqs <- list2(
+    request_test("/status/:status", status = 404),
+    request("INVALID"),
+  )
+  out <- req_perform_sequential(reqs, on_error = "continue")
+  expect_s3_class(out[[1]], "httr2_http_404")
+  expect_s3_class(out[[2]], "httr2_failure")
+})
+
+test_that("on_error = 'return' returns error", {
+  reqs <- list2(
+    request_test("/status/:status", status = 200),
+    request_test("/status/:status", status = 200),
+    request_test("/status/:status", status = 404),
+    request_test("/status/:status", status = 200)
+  )
+  out <- req_perform_sequential(reqs, on_error = "return")
+  expect_length(out, 4)
+  expect_s3_class(out[[3]], "httr2_http_404")
+  expect_equal(out[[4]], NULL)
+})