From c5b037bdf983691b7e44f4f1894735bde3a1fb8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Cs=C3=A1rdi?= Date: Thu, 12 Dec 2019 10:28:41 +0000 Subject: [PATCH 1/9] Use newer roxygen, no R6 yet --- DESCRIPTION | 4 ++-- man/cranlike_metadata_cache.Rd | 8 +------- man/package_cache.Rd | 10 ++-------- man/pkg_cache_api.Rd | 3 +-- 4 files changed, 6 insertions(+), 19 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 6e3023d5..a1ab14af 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -47,5 +47,5 @@ Suggests: testthat, Depends: R (>= 3.1) -Roxygen: list(markdown = TRUE) -RoxygenNote: 6.1.1 +Roxygen: list(markdown = TRUE, r6 = FALSE) +RoxygenNote: 7.0.2 diff --git a/man/cranlike_metadata_cache.Rd b/man/cranlike_metadata_cache.Rd index d34e8258..33744ff7 100644 --- a/man/cranlike_metadata_cache.Rd +++ b/man/cranlike_metadata_cache.Rd @@ -1,13 +1,8 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/metadata-cache.R -\docType{data} \name{cranlike_metadata_cache} \alias{cranlike_metadata_cache} \title{Metadata cache for a CRAN-like repository} -\format{An object of class \code{R6ClassGenerator} of length 24.} -\usage{ -cranlike_metadata_cache -} \description{ This is an R6 class that implements the metadata cache of a CRAN-like repository. For a higher level interface, see the \code{\link[=meta_cache_list]{meta_cache_list()}}, @@ -22,7 +17,7 @@ The cache has several layers: This ensures that the same data is used for all queries of a \code{cranlike_metadata_cache} object. \item It is stored in an RDS file in the user's cache directory. -\item The downloaded raw \code{PACKAGES*} files are cached, together with HTTP +\item The downloaded raw \verb{PACKAGES*} files are cached, together with HTTP ETags, to minimize downloads. } @@ -182,4 +177,3 @@ cmc$deps("pkgconfig") cmc$revdeps("pkgconfig", recursive = FALSE) } } -\keyword{datasets} diff --git a/man/package_cache.Rd b/man/package_cache.Rd index 96181b3d..38612554 100644 --- a/man/package_cache.Rd +++ b/man/package_cache.Rd @@ -1,13 +1,8 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/package-cache.R -\docType{data} \name{package_cache} \alias{package_cache} \title{A simple package cache} -\format{An object of class \code{R6ClassGenerator} of length 24.} -\usage{ -package_cache -} \description{ This is an R6 class that implements a concurrency safe package cache. } @@ -112,8 +107,8 @@ avoid unnecessary downloads. pc <- package_cache$new(path = tempfile()) pc$list() -cat("foo\\n", file = f1 <- tempfile()) -cat("bar\\n", file = f2 <- tempfile()) +cat("foo\n", file = f1 <- tempfile()) +cat("bar\n", file = f2 <- tempfile()) pc$add(f1, "/f1") pc$add(f2, "/f2") pc$list() @@ -121,4 +116,3 @@ pc$find(path = "/f1") pc$copy_to(target = f3 <- tempfile(), path = "/f1") readLines(f3) } -\keyword{datasets} diff --git a/man/pkg_cache_api.Rd b/man/pkg_cache_api.Rd index c2c0e4b3..f010effb 100644 --- a/man/pkg_cache_api.Rd +++ b/man/pkg_cache_api.Rd @@ -19,8 +19,7 @@ pkg_cache_get_file(cachepath = NULL, target, ...) pkg_cache_delete_files(cachepath = NULL, ...) -pkg_cache_add_file(cachepath = NULL, file, relpath = dirname(file), - ...) +pkg_cache_add_file(cachepath = NULL, file, relpath = dirname(file), ...) } \arguments{ \item{cachepath}{Path of the cache. By default the cache directory is in From 0f00f7d800742839d8f0f497bded3f1fe93f0bbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Cs=C3=A1rdi?= Date: Thu, 12 Dec 2019 10:31:12 +0000 Subject: [PATCH 2/9] Update embedded async --- R/aaa-async.R | 289 +++++++++++++++++++++++++++++++++++- tests/async/test-http.R | 22 +++ tests/async/test-when-any.R | 6 +- 3 files changed, 311 insertions(+), 6 deletions(-) diff --git a/R/aaa-async.R b/R/aaa-async.R index b16d20f1..93ef6a70 100644 --- a/R/aaa-async.R +++ b/R/aaa-async.R @@ -2385,9 +2385,10 @@ async_reject <- mark_as_async(async_reject) #' } http_get <- function(url, headers = character(), file = NULL, - options = list(timeout = 600), on_progress = NULL) { + options = list(), on_progress = NULL) { url; headers; file; options; on_progress + options <- get_default_curl_options(options) make_deferred_http( function() { @@ -2447,9 +2448,10 @@ http_get <- mark_as_async(http_get) #' } http_head <- function(url, headers = character(), file = NULL, - options = list(timeout = 600), on_progress = NULL) { + options = list(), on_progress = NULL) { url; headers; file; options; on_progress + options <- get_default_curl_options(options) make_deferred_http( function() { @@ -2472,7 +2474,9 @@ http_head <- mark_as_async(http_head) #' via a deferred value. #' #' @inheritParams http_get -#' @param on_progress: Progress handler function. It is only used if the +#' @param data Data to send. Either a raw vector, or a character string +#' that will be converted to raw with [base::charToRaw]. +#' @param on_progress Progress handler function. It is only used if the #' response body is written to a file. See details at [http_get()]. #' #' @noRd @@ -2491,10 +2495,11 @@ http_head <- mark_as_async(http_head) #' synchronise(do()) http_post <- function(url, data, headers = character(), file = NULL, - options = list(timeout = 600), on_progress = NULL) { + options = list(), on_progress = NULL) { url; data; headers; file; options; on_progress if (!is.raw(data)) data <- charToRaw(data) + options <- get_default_curl_options(options) make_deferred_http( function() { @@ -2512,6 +2517,21 @@ http_post <- function(url, data, headers = character(), file = NULL, http_post <- mark_as_async(http_post) +get_default_curl_options <- function(options) { + getopt <- function(nm) { + if (!is.null(v <- options[[nm]])) return(v) + anm <- paste0("async_http_", nm) + if (!is.null(v <- getOption(anm))) return(v) + if (!is.na(v <- Sys.getenv(toupper(anm), NA_character_))) return (v) + } + list( + timeout = as.integer(getopt("timeout") %||% 3600), + connecttimeout = as.integer(getopt("connecttimeout") %||% 30), + low_speed_time = as.integer(getopt("low_speed_time") %||% 30), + low_speed_limit = as.integer(getopt("low_speed_limit") %||% 100) + ) +} + make_deferred_http <- function(cb, file) { cb; file id <- NULL @@ -2676,6 +2696,262 @@ http_statuses <- c( "599" = "Network connect timeout error (Unknown)" ) +# # Standalone file for controlling when and where to build vignettes ---- +# +# The canonical location of this file is in the asciicast package: +# https://github.com/r-lib/asciicast/master/R/lazyrmd.R +# +# This standalone file provides a vignette builder that gives you more +# control about when, where and how the vignettes of an R package will be +# built. Possible use cases: +# * do not rebuild vignettes on CRAN +# * do not rebuild vignettes on a CI if the build time dependencies are +# not available. +# * avoid rebuilding long running vignettes. +# * fully static vignettes that never rebuild. +# * etc. +# +# ## API +# +# ``` +# onload_hook(local = "if-newer", ci = TRUE, cran = "no-code") +# build() +# build_if_newer() +# ``` +# +# ## Usage +# +# To use this standalone in your package, do the following: +# 1. Copy this file into your package, without changes. +# 2. Call `lazyrmd$onload_hook()` from the `.onLoad()` function of your +# package. For example: +# ``` +# .onLoad <- function(libname, pkgname) { +# lazyrmd$onload_hook( +# local = FALSE, +# ci = function() is_recording_supported(), +# cran = "no-code" +# ) +# } +# ``` +# 3. Add the package itself as a vignette builder, in `DESCRIPTION`. +# 4. Also in `DESCRIPTION`, add knitr and rmarkdown to `Suggests`, if you +# want to install them automatically on the CI, and/or you want to +# build vignettes in CRAN. +# 5. Change the builder of the vignettes that you want to build lazily, +# in the YAML header. E.g.: +# ``` +# vignette: > +# %\VignetteIndexEntry{asciicast example vignette} +# %\VignetteEngine{asciicast::lazyrmd} +# %\VignetteEncoding{UTF-8} +# ``` +# +# ## NEWS: +# +# ### 1.0.0 -- 2019-12-01 +# +# * First release. + +lazyrmd <- local({ + + # config --------------------------------------------------------------- + + # Need to do this before we mess up the environment + package_name <- utils::packageName() + lazy_env <- environment() + parent.env(lazy_env) <- baseenv() + + re_lazy_rmd <- "[.]Rmd$" + + config = list() + + # API ------------------------------------------------------------------ + + #' Configure the lazy vignette builder + #' + #' @noRd + #' @param local Whether/when to build vignettes for local builds. + #' @param ci Whether/when to build vignettes on the CI(s). + #' @param cran Whether/when to build vignettes on CRAN. + #' + #' Possible values for `local`, `ci` and `cran` are: + #' * `TRUE`: always (re)build the vignettes. + #' * `FALSE`: never (re)build the vignettes. + #' * `"if-newer"`: Only re(build) the vignettes if the Rmd file is newer. + #' * `"no-code"`: never rebuild the vignettes, and make sure that the + #' `.R` file only contains dummy code. (This is to make + #' sure that the `.R` file runs on CRAN.) + #' * a function object. This is called without arguments and if it + #' returns an `isTRUE()` value, then the vignette is built. + + onload_hook <- function(local = "if-newer", ci = TRUE, cran = "no-code") { + config <<- list(local = local, ci = ci, cran = cran, forced = NULL) + tools::vignetteEngine( + "lazyrmd", + weave = lazy_weave, + tangle = lazy_tangle, + pattern = re_lazy_rmd, + package = package_name + ) + } + + #' Build vignettes + #' + #' `build()` and `build_if_newer()` are is utility functions, + #' and you would probably not use them in the package code itself. + #' `build()` always rebuilds all vignettes, `build_if_newer()` only + #' if the source file is newer. + #' + #' @return The return value of [tools::buildVignettes()]. + + build_if_newer <- function() { + build_internal("if-newer") + } + + build <- function() { + build_internal("force") + } + + build_internal <- function(mode = c("force", "if-newer")) { + mode <- match.arg(mode) + config$forced <<- if (mode == "force") TRUE else mode + env_save <- Sys.getenv("LAZY_RMD_FORCED", NA_character_) + if (is.na(env_save)) { + on.exit(Sys.unsetenv("LAZY_RMD_FORCED"), add = TRUE) + } else { + on.exit(Sys.setenv(LAZY_RMD_FORCED = env_save), add = TRUE) + } + Sys.setenv(LAZY_RMD_FORCED = "true") + tools::buildVignettes(dir = ".", tangle = TRUE, clean = FALSE) + } + + # internals ------------------------------------------------------------ + + #' This is called by [tools::buildVignettes], as the weave function of + #' the lazyrmd vignette engine + + lazy_weave <- function(file, ...) { + output <- sub(re_lazy_rmd, ".html", file) + env <- detect_env() + conf <- config[[env]] + + if (! should_build(file, output, conf)) { + message("--- chose not to rebuild vignette now") + create_if_needed(output, conf) + return(output) + } + + message("--- weaving vignette...") + tic <- Sys.time() + ret <- weave(file, ...) + toc <- Sys.time() + message("--- weaving vignette... done in ", format(toc - tic)) + ret + } + + weave <- function(file, driver, syntax, encoding = "UTF-8", + quiet = TRUE, ...) { + + opts <- options(markdown.HTML.header = NULL) + on.exit({ + knitr::opts_chunk$restore() + knitr::knit_hooks$restore() + options(opts) + }, add = TRUE) + + knitr::opts_chunk$set(error = FALSE) + + rmarkdown::render( + file, + encoding = encoding, + quiet = quiet, + envir = globalenv(), + ... + ) + } + + #' This is called by [tools::buildVignettes], as the tangle function of + #' the lazyrmd vignette engine + + lazy_tangle <- function(file, ...) { + output <- sub(re_lazy_rmd, ".R", file) + env <- detect_env() + conf <- config[[env]] + + if (! should_build(file, output, conf)) { + create_if_needed(output, conf) + return(output) + } + + message("--- tangling vignette...") + tic <- Sys.time() + ret <- tangle(file, ...) + toc <- Sys.time() + message("--- tangling vignette... done in ", format(toc - tic)) + ret + } + + tangle <- function(file, ..., encoding = "UTF-8", quiet = FALSE) { + output <- sub(re_lazy_rmd, ".R", file) + knitr::purl(file, encoding = encoding, quiet = quiet, ...) + } + + is_ci <- function() { + isTRUE(as.logical(Sys.getenv("CI"))) + } + + #' If NOT_CRAN is set to a true value, then we are local + #' Otherwise we are local if not running inside `R CMD check`. + + is_local <- function() { + if (isTRUE(as.logical(Sys.getenv("NOT_CRAN")))) return(TRUE) + Sys.getenv("_R_CHECK_PACKAGE_NAME_", "") == "" + } + + detect_env <- function() { + # Order matters here, is_local() == TRUE on the CI as well, usually + if (!is.na(Sys.getenv("LAZY_RMD_FORCED", NA_character_))) return("forced") + if (is_ci()) return("ci") + if (is_local()) return("local") + "cran" + } + + should_build <- function(input, output, conf) { + if (isTRUE(conf)) return(TRUE) + if (identical(conf, FALSE)) return(FALSE) + if (is.function(conf)) return(isTRUE(conf())) + if (identical(conf, "if-newer")) { + return(!file.exists(output) || file_mtime(input) > file_mtime(output)) + } + FALSE + } + + file_mtime <- function(...) { + file.info(..., extra_cols = FALSE)$mtime + } + + create_if_needed <- function(path, conf) { + if (!file.exists(path)) file.create(path) + if (identical(conf, "no-code")) { + code_file <- sub("\\.html$", ".R", path) + message("--- creating dummy file ", basename(path)) + cat("# Dummy file for static vignette\n", file = code_file) + } + Sys.setFileTime(path, Sys.time()) + } + + structure( + list( + .internal = lazy_env, + onload_hook = onload_hook, + build = build, + build_if_newer = build_if_newer + ), + class = c("standalone_lazyrmd", "standalone") + ) +}) + #' Apply an asynchronous function to each element of a vector #' #' @param .x A list or atomic vector. @@ -2744,6 +3020,11 @@ async_map_limit <- function(.x, .f, ..., .args = list(), .limit = Inf) { ## nocov start .onLoad <- function(libname, pkgname) { + lazyrmd$onload_hook( + local = "if-newer", + ci = TRUE, + cran = FALSE + ) if (requireNamespace("debugme", quietly = TRUE)) debugme::debugme() } diff --git a/tests/async/test-http.R b/tests/async/test-http.R index 02d1ba92..9e4eef32 100644 --- a/tests/async/test-http.R +++ b/tests/async/test-http.R @@ -237,6 +237,28 @@ test_that("timeout, failed request", { expect_true(toc - tic < as.difftime(4, units = "secs")) }) +test_that("more sophisticated timeouts", { + + skip_if_offline() + + do <- function() { + withr::local_options(list( + async_http_timeout = 6, + async_http_low_speed_time = 2, + async_http_low_speed_limit = 10 + )) + http_get("https://eu.httpbin.org/drip?duration=5&numbytes=10&code=200&delay=0") + } + + tic <- Sys.time() + err <- tryCatch(synchronise(do()), error = identity) + toc <- Sys.time() + + expect_s3_class(err, "async_rejected") + expect_match(conditionMessage(err), "too slow") + expect_true(toc - tic < as.difftime(5, units = "secs")) +}) + test_that("errors contain the response", { skip_if_offline() diff --git a/tests/async/test-when-any.R b/tests/async/test-when-any.R index f2fe5dcc..adcc4e08 100644 --- a/tests/async/test-when-any.R +++ b/tests/async/test-when-any.R @@ -59,15 +59,17 @@ test_that("when_any, late error is ignored", { }) test_that("when_any, multiple errors", { + errors <- list() do <- async(function() { d1 <- delay(1/100 )$then(function(value) stop("foo")) d2 <- delay(1/10000)$then(function(value) stop("bar")) dx <- when_any(d1, d2)$ catch(error = function(reason) { - expect_match(conditionMessage(reason$errors[[1]]), "bar") - expect_match(conditionMessage(reason$errors[[2]]), "foo") + errors <<- reason$errors }) }) synchronise(do()) + expect_match(conditionMessage(errors[[1]]), "bar") + expect_match(conditionMessage(errors[[2]]), "foo") }) From 18a5546b0d3bfe178734693f666366d5f0ffa2b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Cs=C3=A1rdi?= Date: Thu, 12 Dec 2019 10:37:49 +0000 Subject: [PATCH 3/9] Fix test case for new async timeouts --- tests/testthat/test-cran-metadata.R | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/testthat/test-cran-metadata.R b/tests/testthat/test-cran-metadata.R index efe5779e..360d9ea4 100644 --- a/tests/testthat/test-cran-metadata.R +++ b/tests/testthat/test-cran-metadata.R @@ -4,7 +4,13 @@ context("CRAN metadata") test_that("what if cran.r-pkg.org is down?", { skip_if_offline() - withr::with_envvar(c("R_PKG_CRAN_METADATA_URL" = "http://192.0.2.0/"), { + env <- c( + "R_PKG_CRAN_METADATA_URL" = "http://192.0.2.0/", + "ASYNC_HTTP_TIMEOUT" = "1", + "ASYNC_HTTP_CONNECTTIMEOUT" = "1" + ) + + withr::with_envvar(env, { dir.create(pri <- fs::path_norm(tempfile())) on.exit(unlink(pri, recursive = TRUE), add = TRUE) dir.create(rep <- fs::path_norm(tempfile())) From a2e343b021c53223f9eff0c50dbc961ae7345172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Cs=C3=A1rdi?= Date: Thu, 12 Dec 2019 10:45:40 +0000 Subject: [PATCH 4/9] Add pak project --- .Rbuildignore | 2 ++ .Rprofile | 13 +++++++++++++ .gitignore | 1 + 3 files changed, 16 insertions(+) create mode 100644 .Rprofile diff --git a/.Rbuildignore b/.Rbuildignore index 7e27007e..ed4d2148 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -7,3 +7,5 @@ ^LICENSE\.md$ ^tools$ ^revdep$ +^\.Rprofile$ +^r-packages$ diff --git a/.Rprofile b/.Rprofile new file mode 100644 index 00000000..a4429246 --- /dev/null +++ b/.Rprofile @@ -0,0 +1,13 @@ +# This file is created automatically by `pak:::proj_create()` +# Please do not edit this file +{ + cat("-> Setting up private project library.\n") + .libPaths(unique(c(file.path("r-packages", R.version$platform, + getRversion()[, 1:2]), .libPaths()))) + if (file.exists("~/.Rprofile")) + source("~/.Rprofile") + if (interactive()) { + cat("-> Call `pak:::proj_install_dev()` to install/update dependencies.\n") + cat("-> Call `pak:::proj_load()` to load the package.\n") + } +} diff --git a/.gitignore b/.gitignore index 66dc678a..64b986d3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .Rproj.user /README.html /revdep +/r-packages From 87dcbe4a59ae816e1e3d7660a695b70aac835ff3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Cs=C3=A1rdi?= Date: Thu, 12 Dec 2019 11:04:03 +0000 Subject: [PATCH 5/9] Update embedded async --- NAMESPACE | 1 + R/aaa-async.R | 15 ++++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index dc82ce83..8ac28eba 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -55,6 +55,7 @@ importFrom(utils,getSrcFilename) importFrom(utils,getSrcLocation) importFrom(utils,head) importFrom(utils,installed.packages) +importFrom(utils,modifyList) importFrom(utils,read.csv) importFrom(uuid,UUIDgenerate) importFrom(withr,defer) diff --git a/R/aaa-async.R b/R/aaa-async.R index 93ef6a70..7629448f 100644 --- a/R/aaa-async.R +++ b/R/aaa-async.R @@ -2517,6 +2517,8 @@ http_post <- function(url, data, headers = character(), file = NULL, http_post <- mark_as_async(http_post) +#' @importFrom utils modifyList + get_default_curl_options <- function(options) { getopt <- function(nm) { if (!is.null(v <- options[[nm]])) return(v) @@ -2524,11 +2526,14 @@ get_default_curl_options <- function(options) { if (!is.null(v <- getOption(anm))) return(v) if (!is.na(v <- Sys.getenv(toupper(anm), NA_character_))) return (v) } - list( - timeout = as.integer(getopt("timeout") %||% 3600), - connecttimeout = as.integer(getopt("connecttimeout") %||% 30), - low_speed_time = as.integer(getopt("low_speed_time") %||% 30), - low_speed_limit = as.integer(getopt("low_speed_limit") %||% 100) + modifyList( + options, + list( + timeout = as.integer(getopt("timeout") %||% 3600), + connecttimeout = as.integer(getopt("connecttimeout") %||% 30), + low_speed_time = as.integer(getopt("low_speed_time") %||% 30), + low_speed_limit = as.integer(getopt("low_speed_limit") %||% 100) + ) ) } From e89856b09d97ded0c3bba58f0ed61a497ea2bd17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Cs=C3=A1rdi?= Date: Thu, 12 Dec 2019 10:46:04 +0000 Subject: [PATCH 6/9] Use our own timeouts In case they change in async --- R/async-http.R | 53 +++++++++++++++++++++-------- tests/testthat/test-cran-metadata.R | 4 +-- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/R/async-http.R b/R/async-http.R index 08d80dbe..a62c6546 100644 --- a/R/async-http.R +++ b/R/async-http.R @@ -1,4 +1,21 @@ +#' @importFrom utils modifyList + +get_async_timeouts <- function(options) { + getopt <- function(nm) { + if (!is.null(v <- options[[nm]])) return(v) + anm <- paste0("pkgcache_", nm) + if (!is.null(v <- getOption(anm))) return(v) + if (!is.na(v <- Sys.getenv(toupper(anm), NA_character_))) return (v) + } + list( + timeout = as.integer(getopt("timeout") %||% 3600), + connecttimeout = as.integer(getopt("connecttimeout") %||% 30), + low_speed_time = as.integer(getopt("low_speed_time") %||% 30), + low_speed_limit = as.integer(getopt("low_speed_limit") %||% 100) + ) +} + #' Download a file, asynchronously #' #' This is the asynchronous version of [utils::download.file()]. @@ -18,6 +35,7 @@ #' It can be used to cache the file, with the [download_if_newer()] or #' the [download_one_of()] functions. #' @param tmp_destfile Where to store the temporary destination file. +#' @param options Curl options. #' @param ... Additional arguments are passed to [http_get()]. #' @return A [deferred] object. It resolves to a list with entries: #' * `url`: The URL in the request. @@ -59,20 +77,23 @@ #' ``` download_file <- function(url, destfile, etag_file = NULL, - tmp_destfile = paste0(destfile, ".tmp"), ...) { + tmp_destfile = paste0(destfile, ".tmp"), + options = list(), ...) { "!DEBUG downloading `url`" assert_that( is_string(url), is_path(destfile), is_path(tmp_destfile), - is_path_or_null(etag_file)) + is_path_or_null(etag_file), + is.list(options)) force(list(...)) + options <- get_async_timeouts(options) destfile <- normalizePath(destfile, mustWork = FALSE) tmp_destfile <- normalizePath(tmp_destfile, mustWork = FALSE) mkdirp(dirname(tmp_destfile)) - http_get(url, file = tmp_destfile, ...)$ + http_get(url, file = tmp_destfile, options = options, ...)$ then(http_stop_for_status)$ then(function(resp) { "!DEBUG downloaded `url`" @@ -169,7 +190,7 @@ get_etag_header_from_file <- function(destfile, etag_file) { download_if_newer <- function(url, destfile, etag_file = NULL, headers = NULL, tmp_destfile = paste0(destfile, ".tmp"), - ...) { + options = list(), ...) { "!DEBUG download if newer `url`" headers <- headers %||% structure(character(), names = character()) assert_that( @@ -177,9 +198,11 @@ download_if_newer <- function(url, destfile, etag_file = NULL, is_path(destfile), is_path(tmp_destfile), is_path_or_null(etag_file), - is.character(headers), all_named(headers)) + is.character(headers), all_named(headers), + is.list(options)) force(list(...)) + options <- get_async_timeouts(options) etag_old <- get_etag_header_from_file(destfile, etag_file) headers <- c(headers, etag_old) @@ -187,7 +210,8 @@ download_if_newer <- function(url, destfile, etag_file = NULL, tmp_destfile <- normalizePath(tmp_destfile, mustWork = FALSE) mkdirp(dirname(tmp_destfile)) - http_get(url, file = tmp_destfile, headers = headers, ...)$ + http_get(url, file = tmp_destfile, headers = headers, + options = options, ...)$ then(http_stop_for_status)$ then(function(resp) { if (resp$status_code == 304) { @@ -278,21 +302,23 @@ download_if_newer <- function(url, destfile, etag_file = NULL, #' ``` download_one_of <- function(urls, destfile, etag_file = NULL, - headers = NULL, ...) { + headers = NULL, options = list(), ...) { "!DEBUG trying multiple URLs" headers <- headers %||% structure(character(), names = character()) assert_that( is_character(urls), length(urls) >= 1, is_path(destfile), is_path_or_null(etag_file), - is.character(headers), all_named(headers)) + is.character(headers), all_named(headers), + is.list(options)) force(list(...)) + options <- get_async_timeouts(options) tmps <- paste0(destfile, ".tmp.", seq_along(urls)) dls <- mapply( download_if_newer, url = urls, tmp_destfile = tmps, MoreArgs = list(destfile = destfile, etag_file = etag_file, - headers = headers, ...), + headers = headers, options = options, ...), SIMPLIFY = FALSE) when_any(.list = dls)$ @@ -303,26 +329,25 @@ download_one_of <- function(urls, destfile, etag_file = NULL, }) } -download_files <- function(data, ...) { +download_files <- function(data, options = list(), ...) { if (any(dup <- duplicated(data$path))) { stop("Duplicate target paths in download_files: ", paste0("`", unique(data$path[dup]), "`", collapse = ", "), ".") } + options <- get_async_timeouts(options) bar <- create_progress_bar(data) prog_cb <- function(upd) update_progress_bar_progress(bar, upd) dls <- lapply(seq_len(nrow(data)), function(idx) { row <- data[idx, ] dx <- download_if_newer(row$url, row$path, row$etag, - on_progress = prog_cb, options = list(timeout = row$timeout %||% 100), - ...) + on_progress = prog_cb, options = options, ...) if ("fallback_url" %in% names(row) && !is.na(row$fallback_url)) { dx <- dx$catch(error = function(err) { download_if_newer(row$fallback_url, row$path, row$etag, - options = list(timeout = row$timeout %||% 10), - ...) + options = options, ...) }) } diff --git a/tests/testthat/test-cran-metadata.R b/tests/testthat/test-cran-metadata.R index 360d9ea4..96472ba0 100644 --- a/tests/testthat/test-cran-metadata.R +++ b/tests/testthat/test-cran-metadata.R @@ -6,8 +6,8 @@ test_that("what if cran.r-pkg.org is down?", { env <- c( "R_PKG_CRAN_METADATA_URL" = "http://192.0.2.0/", - "ASYNC_HTTP_TIMEOUT" = "1", - "ASYNC_HTTP_CONNECTTIMEOUT" = "1" + "PKGCACHE_TIMEOUT" = "1", + "PKGCACHE_CONNECTTIMEOUT" = "1" ) withr::with_envvar(env, { From 1bfd2619bc8a7453d8bbcd2bf2d5b9de90ed9185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Cs=C3=A1rdi?= Date: Thu, 12 Dec 2019 11:35:26 +0000 Subject: [PATCH 7/9] Create ?pkgcache page, share with README --- .Rbuildignore | 1 + .gitignore | 1 + R/package.R | 6 ++ README.Rmd | 92 +------------------ README.md | 128 +++++++++++++++++--------- man/pkgcache-package.Rd | 198 ++++++++++++++++++++++++++++++++++++++++ tools/README-body.Rmd | 127 ++++++++++++++++++++++++++ 7 files changed, 423 insertions(+), 130 deletions(-) create mode 100644 man/pkgcache-package.Rd create mode 100644 tools/README-body.Rmd diff --git a/.Rbuildignore b/.Rbuildignore index ed4d2148..c20c24c1 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -9,3 +9,4 @@ ^revdep$ ^\.Rprofile$ ^r-packages$ +^.*_cache$ diff --git a/.gitignore b/.gitignore index 64b986d3..f1731409 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ /README.html /revdep /r-packages +*_cache diff --git a/R/package.R b/R/package.R index e69de29b..826212c6 100644 --- a/R/package.R +++ b/R/package.R @@ -0,0 +1,6 @@ + +#' Cache for package data and metadata +#' +#' @includeRmd tools/README-body.Rmd +#' +"_PACKAGE" diff --git a/README.Rmd b/README.Rmd index 3bc29845..01d51c61 100644 --- a/README.Rmd +++ b/README.Rmd @@ -9,9 +9,11 @@ knitr::opts_chunk$set( collapse = TRUE, comment = "#>", fig.path = "man/figures/README-", - out.width = "100%" + out.width = "100%", + cache = TRUE ) ``` + # pkgcache > Cache CRAN-like metadata and package files @@ -22,91 +24,5 @@ knitr::opts_chunk$set( [![AppVeyor build status](https://ci.appveyor.com/api/projects/status/github/r-lib/pkgcache?branch=master&svg=true)](https://ci.appveyor.com/project/r-lib/pkgcache) [![Coverage status](https://codecov.io/gh/r-lib/pkgcache/branch/master/graph/badge.svg)](https://codecov.io/github/r-lib/pkgcache?branch=master) -Metadata and package cache for CRAN-like repositories. This is a utility -package to be used by package management tools that want to take advantage -of caching. - -## Installation - -You can install the released version of pkgcache from -[CRAN](https://CRAN.R-project.org) with: - -``` r -install.packages("pkgcache") -``` - -## Metadata cache - -`meta_cache_list()` lists all packages in the metadata cache. It includes -Bioconductor package, and all versions (i.e. both binary and source) -of the packages for the current platform and R version. - -```{r} -library(pkgcache) -meta_cache_list() -``` - -`meta_cache_deps()` and `meta_cache_revdeps()` can be used to look up -dependencies and reverse dependencies. - -The metadata is updated automatically if it is older than seven days, -and it can also be updated manually with `meta_cache_update()`. - -See the `cranlike_metadata_cache` R6 class for a lower level API, -and more control. - -## Package cache - -Package management tools may use the `pkg_cache_*` functions and in -particular the `package_cache` class, to make use of local caching of -package files. - -The `pkg_cache_*` API is high level, and uses a user level cache: - -```{r} -pkg_cache_summary() +```{r child = "tools/README-body.Rmd"} ``` - -```{r} -pkg_cache_list() -``` - -```{r} -pkg_cache_find(package = "dplyr") -``` - -`pkg_cache_add_file()` can be used to add a file, -`pkg_cache_delete_files()` to remove files, `pkg_cache_get_files()` to -copy files out of the cache. - -The `package_cache` class provides a finer API. - -## Bioconductor support - -Both the metadata cache and the package cache support Bioconductor by -default, automatically. The following options and environment variables -can be used to configure pkgcache's Bioconductor support: - -Options: - -- The `BioC_mirror` option can be used to select a Bioconductor mirror. - This takes priority over the `R_BIOC_MIRROR` environment variable. - -Environment variables: - -- The `R_BIOC_VERSION` environment variable can be used to override the - default Bioconductor version detection and force a given version. E.g. - this can be used to force the development version of Bioconductor. -- The `R_BIOC_MIRROR` environment variable can be used to select a - Bioconductor mirror. The `BioC_mirror` option takes priority over this, - if set. - -## Code of Conduct - -Please note that the 'pkgcache' project is released with a -[Contributor Code of Conduct](https://github.com/r-lib/pkgcache/blob/master/.github/CODE_OF_CONDUCT.md). -By contributing to this project, you agree to abide by its terms. - -## License - -MIT © [RStudio Inc](https://rstudio.com) diff --git a/README.md b/README.md index 5c5efd47..5b5e2502 100644 --- a/README.md +++ b/README.md @@ -37,27 +37,26 @@ source) of the packages for the current platform and R version. ``` r library(pkgcache) meta_cache_list() -#> ✔ Using cached package metadata -#> # A tibble: 32,705 x 33 +#> # A tibble: 35,272 x 33 #> package title version depends suggests built imports archs repodir #> #> 1 A3 "Acc… 1.0.0 R (>= … randomF… R 3.… bin/ma… -#> 2 abbyyR Acce… 0.5.4 R (>= … testtha… R 3.… httr, … bin/ma… -#> 3 abc.da… Data… 1.0 R (>= … R 3.… bin/ma… -#> 4 ABC.RAP Arra… 0.9.0 R (>= … knitr, … R 3.… graphi… bin/ma… -#> 5 abc Tool… 2.1 R (>= … R 3.… bin/ma… -#> 6 ABCana… Comp… 1.2.1 R (>= … R 3.… plotrix bin/ma… -#> 7 abcdeF… "ABC… 0.4 Rglpk,… LIM,syb… R 3.… bin/ma… -#> 8 ABCopt… Impl… 0.15.0 testtha… R 3.… Rcpp, … ABCo… bin/ma… -#> 9 ABCp2 Appr… 1.2 MASS R 3.… bin/ma… -#> 10 abcrf Appr… 1.7.1 R(>= 3… R 3.… "readr… abcr… bin/ma… -#> # … with 32,695 more rows, and 24 more variables: platform , +#> 2 aaSEA Amin… 1.1.0 R(>= 3… knitr, … R 3.… "DT(>=… bin/ma… +#> 3 ABACUS "App… 1.0.0 R (>= … rmarkdo… R 3.… ggplot… bin/ma… +#> 4 abbyyR Acce… 0.5.5 R (>= … testtha… R 3.… httr, … bin/ma… +#> 5 abc.da… Data… 1.0 R (>= … R 3.… bin/ma… +#> 6 ABC.RAP Arra… 0.9.0 R (>= … knitr, … R 3.… graphi… bin/ma… +#> 7 abc Tool… 2.1 R (>= … R 3.… bin/ma… +#> 8 abcADM Fit … 1.0 R 3.… Rcpp (… abcA… bin/ma… +#> 9 ABCana… Comp… 1.2.1 R (>= … R 3.… plotrix bin/ma… +#> 10 abcdeF… "ABC… 0.4 Rglpk,… LIM,syb… R 3.… bin/ma… +#> # … with 35,262 more rows, and 24 more variables: platform , #> # rversion , needscompilation , priority , ref , #> # type , direct , status , target , mirror , #> # sources , filesize , sha256 , sysreqs , -#> # published , deps , license , linkingto , -#> # enhances , license_restricts_use , os_type , -#> # license_is_foss , md5sum , path +#> # published , deps , license , md5sum , +#> # linkingto , enhances , license_restricts_use , +#> # os_type , license_is_foss , path ``` `meta_cache_deps()` and `meta_cache_revdeps()` can be used to look up @@ -83,36 +82,40 @@ pkg_cache_summary() #> [1] "/Users/gaborcsardi/Library/Caches/R-pkg/pkg" #> #> $files -#> [1] 12 +#> [1] 677 #> #> $size -#> [1] 553116 +#> [1] 608180353 ``` ``` r pkg_cache_list() -#> # A tibble: 12 x 8 -#> fullpath path package url etag sha256 built vignettes -#> -#> 1 /Users/gaborc… src/contri… cranca… NA NA 936fa23d… FALSE -#> 2 /Users/gaborc… src/contri… cranca… NA NA 936fa23d… TRUE FALSE -#> 3 /Users/gaborc… src/contri… rcmdch… NA NA 6332ddc2… FALSE -#> 4 /Users/gaborc… src/contri… rcmdch… NA NA 6332ddc2… TRUE FALSE -#> 5 /Users/gaborc… src/contri… revdep… NA NA e204aede… FALSE -#> 6 /Users/gaborc… src/contri… revdep… NA NA e204aede… TRUE FALSE -#> 7 /Users/gaborc… src/contri… cranca… NA NA 936fa23d… FALSE -#> 8 /Users/gaborc… src/contri… cranca… NA NA 936fa23d… TRUE FALSE -#> 9 /Users/gaborc… src/contri… revdep… NA NA e204aede… FALSE -#> 10 /Users/gaborc… src/contri… revdep… NA NA e204aede… TRUE FALSE -#> 11 /Users/gaborc… src/contri… rcmdch… NA NA 6332ddc2… FALSE -#> 12 /Users/gaborc… src/contri… rcmdch… NA NA 6332ddc2… TRUE FALSE +#> # A tibble: 677 x 10 +#> fullpath path package url etag sha256 built version platform +#> +#> 1 /Users/… src/… crayon 84be6… 0 +#> 2 /Users/… bin/… proces… http… "\"3… ce84a… NA 3.4.1 macos +#> 3 /Users/… bin/… R6 http… "\"d… 4cfe3… NA 2.4.0 macos +#> 4 /Users/… bin/… ps http… "\"3… 63add… NA 1.3.0 macos +#> 5 /Users/… src/… callr df85a… 0 +#> 6 /Users/… bin/… callr http… "\"5… 1a5e3… NA 3.3.1 macos +#> 7 /Users/… bin/… assert… http… "\"d… 103f9… NA 0.2.1 macos +#> 8 /Users/… bin/… backpo… http… "\"d… 344bb… NA 1.1.4 macos +#> 9 /Users/… bin/… crayon http… "\"b… 2ac4f… NA 1.3.4 macos +#> 10 /Users/… bin/… desc http… "\"4… 196ce… NA 1.2.0 macos +#> # … with 667 more rows, and 1 more variable: vignettes ``` ``` r pkg_cache_find(package = "dplyr") -#> # A tibble: 0 x 8 -#> # … with 8 variables: fullpath , path , package , -#> # url , etag , sha256 , built , vignettes +#> # A tibble: 4 x 10 +#> fullpath path package url etag sha256 built version platform +#> +#> 1 /Users/… bin/… dplyr http… "\"6… e2594… NA 0.8.3 macos +#> 2 /Users/… src/… dplyr 134b1… 0 +#> 3 /Users/… src/… dplyr b357f… 0 +#> 4 /Users/… src/… dplyr 81767… 0 +#> # … with 1 more variable: vignettes ``` `pkg_cache_add_file()` can be used to add a file, @@ -124,16 +127,34 @@ The `package_cache` class provides a finer API. ## Bioconductor support Both the metadata cache and the package cache support Bioconductor by -default, automatically. The following options and environment variables -can be used to configure pkgcache’s Bioconductor support: +default, automatically. See the `BioC_mirror` option and the +`R_BIOC_MIRROR` and `R_BIOC_VERSION` environment variables below to +configure pkgcache’s Bioconductor support. -Options: +## Package Options - The `BioC_mirror` option can be used to select a Bioconductor mirror. This takes priority over the `R_BIOC_MIRROR` environment variable. - -Environment variables: + - `pkgcache_timeout` is the HTTP timeout for all downloads. It is in + seconds, and the limit for downloading the whole file. Defaults to + 3600, one hour. It corresponds to the [`TIMEOUT` libcurl + option](https://curl.haxx.se/libcurl/c/CURLOPT_TIMEOUT.html). + - `pkgcache_connecttimeout` is the HTTP timeout for the connection + phase. It is in seconds and defaults to 30 seconds. It corresponds + to the [`CONNECTTIMEOUT` libcurl + option](https://curl.haxx.se/libcurl/c/CURLOPT_CONNECTTIMEOUT.html). + - `pkgcache_low_speed_limit` and `pkgcache_low_speed_time` are used + for a more sensible HTTP timeout. If the download speed is less than + `pkgcache_low_speed_limit` bytes per second for at least + `pkgcache_low_speed_time` seconds, the download errors. They + correspond to the + [`LOW_SPEED_LIMIT`](https://curl.haxx.se/libcurl/c/CURLOPT_LOW_SPEED_LIMIT.html) + and + [`LOW_SPEED_TIME`](https://curl.haxx.se/libcurl/c/CURLOPT_LOW_SPEED_TIME.html) + curl options. + +## Package environment variables - The `R_BIOC_VERSION` environment variable can be used to override the default Bioconductor version detection and force a given @@ -142,11 +163,34 @@ Environment variables: - The `R_BIOC_MIRROR` environment variable can be used to select a Bioconductor mirror. The `BioC_mirror` option takes priority over this, if set. + - `PKGCACHE_TIMEOUT` is the HTTP timeout for all downloads. It is in + seconds, and the limit for downloading the whole file. Defaults to + 3600, one hour. It corresponds to the [`TIMEOUT` libcurl + option](https://curl.haxx.se/libcurl/c/CURLOPT_TIMEOUT.html). The + `pkgcache_timeout` option has priority over this, if set. + - `PKGCACHE_CONNECTTIMEOUT` is the HTTP timeout for the connection + phase. It is in seconds and defaults to 30 seconds. It corresponds + to the [`CONNECTTIMEOUT` libcurl + option](https://curl.haxx.se/libcurl/c/CURLOPT_CONNECTTIMEOUT.html). + The `pkgcache_connecttimeout` option takes precedence over this, if + set. + - `PKGCACHE_LOW_SPEED_LIMIT` and `PKGCACHE_LOW_SPEED_TIME` are used + for a more sensible HTTP timeout. If the download speed is less than + `PKGCACHE_LOW_SPEED_LIMIT` bytes per second for at least + `PKGCACHE_LOW_SPEED_TIME` seconds, the download errors. They + correspond to the + [`LOW_SPEED_LIMIT`](https://curl.haxx.se/libcurl/c/CURLOPT_LOW_SPEED_LIMIT.html) + and + [`LOW_SPEED_TIME`](https://curl.haxx.se/libcurl/c/CURLOPT_LOW_SPEED_TIME.html) + curl options. The `pkgcache_low_speed_time` and + `pkgcache_low_speed_limit` options have priority over these + environment variables, if they are set. ## Code of Conduct -Please note that the 'pkgcache' project is released with a -[Contributor Code of Conduct](https://github.com/r-lib/pkgcache/blob/master/.github/CODE_OF_CONDUCT.md). +Please note that the ‘pkgcache’ project is released with a [Contributor +Code of +Conduct](https://github.com/r-lib/pkgcache/blob/master/.github/CODE_OF_CONDUCT.md). By contributing to this project, you agree to abide by its terms. ## License diff --git a/man/pkgcache-package.Rd b/man/pkgcache-package.Rd new file mode 100644 index 00000000..d9d2bb76 --- /dev/null +++ b/man/pkgcache-package.Rd @@ -0,0 +1,198 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/package.R +\docType{package} +\name{pkgcache-package} +\alias{pkgcache} +\alias{pkgcache-package} +\title{Cache for package data and metadata} +\description{ +Metadata and package cache for CRAN-like repositories. + This is a utility package to be used by package management tools + that want to take advantage of caching. +} +\details{ +Metadata and package cache for CRAN-like repositories. This is a utility +package to be used by package management tools that want to take +advantage of caching. +\subsection{Installation}{ + +You can install the released version of pkgcache from +\href{https://CRAN.R-project.org}{CRAN} with:\if{html}{\out{
}}\preformatted{install.packages("pkgcache") +}\if{html}{\out{
}} +} + +\subsection{Metadata cache}{ + +\code{meta_cache_list()} lists all packages in the metadata cache. It +includes Bioconductor package, and all versions (i.e. both binary and +source) of the packages for the current platform and R version.\if{html}{\out{
}}\preformatted{library(pkgcache) +meta_cache_list() +#> +ℹ Loading global cached package metadata +✔ Loading global cached package metadata +#> # A tibble: 35,272 x 33 +#> package title version depends suggests built imports archs repodir +#> +#> 1 A3 "Acc… 1.0.0 R (>= … randomF… R 3.… bin/ma… +#> 2 aaSEA Amin… 1.1.0 R(>= 3… knitr, … R 3.… "DT(>=… bin/ma… +#> 3 ABACUS "App… 1.0.0 R (>= … rmarkdo… R 3.… ggplot… bin/ma… +#> 4 abbyyR Acce… 0.5.5 R (>= … testtha… R 3.… httr, … bin/ma… +#> 5 abc.da… Data… 1.0 R (>= … R 3.… bin/ma… +#> 6 ABC.RAP Arra… 0.9.0 R (>= … knitr, … R 3.… graphi… bin/ma… +#> 7 abc Tool… 2.1 R (>= … R 3.… bin/ma… +#> 8 abcADM Fit … 1.0 R 3.… Rcpp (… abcA… bin/ma… +#> 9 ABCana… Comp… 1.2.1 R (>= … R 3.… plotrix bin/ma… +#> 10 abcdeF… "ABC… 0.4 Rglpk,… LIM,syb… R 3.… bin/ma… +#> # … with 35,262 more rows, and 24 more variables: platform , +#> # rversion , needscompilation , priority , ref , +#> # type , direct , status , target , mirror , +#> # sources , filesize , sha256 , sysreqs , +#> # published , deps , license , md5sum , +#> # linkingto , enhances , license_restricts_use , +#> # os_type , license_is_foss , path +}\if{html}{\out{
}} + +\code{meta_cache_deps()} and \code{meta_cache_revdeps()} can be used to look up +dependencies and reverse dependencies. + +The metadata is updated automatically if it is older than seven days, +and it can also be updated manually with \code{meta_cache_update()}. + +See the \code{cranlike_metadata_cache} R6 class for a lower level API, and +more control. +} + +\subsection{Package cache}{ + +Package management tools may use the \verb{pkg_cache_*} functions and in +particular the \code{package_cache} class, to make use of local caching of +package files. + +The \verb{pkg_cache_*} API is high level, and uses a user level cache:\if{html}{\out{
}}\preformatted{pkg_cache_summary() +#> $cachepath +#> [1] "/Users/gaborcsardi/Library/Caches/R-pkg/pkg" +#> +#> $files +#> [1] 677 +#> +#> $size +#> [1] 608180353 +}\if{html}{\out{
}}\if{html}{\out{
}}\preformatted{pkg_cache_list() +#> # A tibble: 677 x 10 +#> fullpath path package url etag sha256 built version platform +#> +#> 1 /Users/… src/… crayon 84be6… 0 +#> 2 /Users/… bin/… proces… http… "\"3… ce84a… NA 3.4.1 macos +#> 3 /Users/… bin/… R6 http… "\"d… 4cfe3… NA 2.4.0 macos +#> 4 /Users/… bin/… ps http… "\"3… 63add… NA 1.3.0 macos +#> 5 /Users/… src/… callr df85a… 0 +#> 6 /Users/… bin/… callr http… "\"5… 1a5e3… NA 3.3.1 macos +#> 7 /Users/… bin/… assert… http… "\"d… 103f9… NA 0.2.1 macos +#> 8 /Users/… bin/… backpo… http… "\"d… 344bb… NA 1.1.4 macos +#> 9 /Users/… bin/… crayon http… "\"b… 2ac4f… NA 1.3.4 macos +#> 10 /Users/… bin/… desc http… "\"4… 196ce… NA 1.2.0 macos +#> # … with 667 more rows, and 1 more variable: vignettes +}\if{html}{\out{
}}\if{html}{\out{
}}\preformatted{pkg_cache_find(package = "dplyr") +#> # A tibble: 4 x 10 +#> fullpath path package url etag sha256 built version platform +#> +#> 1 /Users/… bin/… dplyr http… "\"6… e2594… NA 0.8.3 macos +#> 2 /Users/… src/… dplyr 134b1… 0 +#> 3 /Users/… src/… dplyr b357f… 0 +#> 4 /Users/… src/… dplyr 81767… 0 +#> # … with 1 more variable: vignettes +}\if{html}{\out{
}} + +\code{pkg_cache_add_file()} can be used to add a file, +\code{pkg_cache_delete_files()} to remove files, \code{pkg_cache_get_files()} to +copy files out of the cache. + +The \code{package_cache} class provides a finer API. +} + +\subsection{Bioconductor support}{ + +Both the metadata cache and the package cache support Bioconductor by +default, automatically. See the \code{BioC_mirror} option and the +\code{R_BIOC_MIRROR} and \code{R_BIOC_VERSION} environment variables below to +configure pkgcache’s Bioconductor support. +} + +\subsection{Package Options}{ +\itemize{ +\item The \code{BioC_mirror} option can be used to select a Bioconductor +mirror. This takes priority over the \code{R_BIOC_MIRROR} environment +variable. +\item \code{pkgcache_timeout} is the HTTP timeout for all downloads. It is in +seconds, and the limit for downloading the whole file. Defaults to +3600, one hour. It corresponds to the \href{https://curl.haxx.se/libcurl/c/CURLOPT_TIMEOUT.html}{\code{TIMEOUT} libcurl option}. +\item \code{pkgcache_connecttimeout} is the HTTP timeout for the connection +phase. It is in seconds and defaults to 30 seconds. It corresponds +to the \href{https://curl.haxx.se/libcurl/c/CURLOPT_CONNECTTIMEOUT.html}{\code{CONNECTTIMEOUT} libcurl option}. +\item \code{pkgcache_low_speed_limit} and \code{pkgcache_low_speed_time} are used +for a more sensible HTTP timeout. If the download speed is less than +\code{pkgcache_low_speed_limit} bytes per second for at least +\code{pkgcache_low_speed_time} seconds, the download errors. They +correspond to the +\href{https://curl.haxx.se/libcurl/c/CURLOPT_LOW_SPEED_LIMIT.html}{\code{LOW_SPEED_LIMIT}} +and +\href{https://curl.haxx.se/libcurl/c/CURLOPT_LOW_SPEED_TIME.html}{\code{LOW_SPEED_TIME}} +curl options. +} +} + +\subsection{Package environment variables}{ +\itemize{ +\item The \code{R_BIOC_VERSION} environment variable can be used to override +the default Bioconductor version detection and force a given +version. E.g. this can be used to force the development version of +Bioconductor. +\item The \code{R_BIOC_MIRROR} environment variable can be used to select a +Bioconductor mirror. The \code{BioC_mirror} option takes priority over +this, if set. +\item \code{PKGCACHE_TIMEOUT} is the HTTP timeout for all downloads. It is in +seconds, and the limit for downloading the whole file. Defaults to +3600, one hour. It corresponds to the \href{https://curl.haxx.se/libcurl/c/CURLOPT_TIMEOUT.html}{\code{TIMEOUT} libcurl option}. The +\code{pkgcache_timeout} option has priority over this, if set. +\item \code{PKGCACHE_CONNECTTIMEOUT} is the HTTP timeout for the connection +phase. It is in seconds and defaults to 30 seconds. It corresponds +to the \href{https://curl.haxx.se/libcurl/c/CURLOPT_CONNECTTIMEOUT.html}{\code{CONNECTTIMEOUT} libcurl option}. +The \code{pkgcache_connecttimeout} option takes precedence over this, if +set. +\item \code{PKGCACHE_LOW_SPEED_LIMIT} and \code{PKGCACHE_LOW_SPEED_TIME} are used +for a more sensible HTTP timeout. If the download speed is less than +\code{PKGCACHE_LOW_SPEED_LIMIT} bytes per second for at least +\code{PKGCACHE_LOW_SPEED_TIME} seconds, the download errors. They +correspond to the +\href{https://curl.haxx.se/libcurl/c/CURLOPT_LOW_SPEED_LIMIT.html}{\code{LOW_SPEED_LIMIT}} +and +\href{https://curl.haxx.se/libcurl/c/CURLOPT_LOW_SPEED_TIME.html}{\code{LOW_SPEED_TIME}} +curl options. The \code{pkgcache_low_speed_time} and +\code{pkgcache_low_speed_limit} options have priority over these +environment variables, if they are set. +} +} + +\subsection{Code of Conduct}{ + +Please note that the ‘pkgcache’ project is released with a \href{https://github.com/r-lib/pkgcache/blob/master/.github/CODE_OF_CONDUCT.md}{Contributor Code of Conduct}. +By contributing to this project, you agree to abide by its terms. +} + +\subsection{License}{ + +MIT © \href{https://rstudio.com}{RStudio Inc} +} +} +\seealso{ +Useful links: +\itemize{ + \item \url{https://github.com/r-lib/pkgcache#readme} + \item Report bugs at \url{https://github.com/r-lib/pkgcache/issues} +} + +} +\author{ +\strong{Maintainer}: Gábor Csárdi \email{csardi.gabor@gmail.com} + +} diff --git a/tools/README-body.Rmd b/tools/README-body.Rmd new file mode 100644 index 00000000..66a738a6 --- /dev/null +++ b/tools/README-body.Rmd @@ -0,0 +1,127 @@ + +```{r setup2, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>", + cache = TRUE +) +``` + +Metadata and package cache for CRAN-like repositories. This is a utility +package to be used by package management tools that want to take advantage +of caching. + +## Installation + +You can install the released version of pkgcache from +[CRAN](https://CRAN.R-project.org) with: + +``` r +install.packages("pkgcache") +``` + +## Metadata cache + +`meta_cache_list()` lists all packages in the metadata cache. It includes +Bioconductor package, and all versions (i.e. both binary and source) +of the packages for the current platform and R version. + +```{r} +library(pkgcache) +meta_cache_list() +``` + +`meta_cache_deps()` and `meta_cache_revdeps()` can be used to look up +dependencies and reverse dependencies. + +The metadata is updated automatically if it is older than seven days, +and it can also be updated manually with `meta_cache_update()`. + +See the `cranlike_metadata_cache` R6 class for a lower level API, +and more control. + +## Package cache + +Package management tools may use the `pkg_cache_*` functions and in +particular the `package_cache` class, to make use of local caching of +package files. + +The `pkg_cache_*` API is high level, and uses a user level cache: + +```{r} +pkg_cache_summary() +``` + +```{r} +pkg_cache_list() +``` + +```{r} +pkg_cache_find(package = "dplyr") +``` + +`pkg_cache_add_file()` can be used to add a file, +`pkg_cache_delete_files()` to remove files, `pkg_cache_get_files()` to +copy files out of the cache. + +The `package_cache` class provides a finer API. + +## Bioconductor support + +Both the metadata cache and the package cache support Bioconductor by +default, automatically. See the `BioC_mirror` option and the `R_BIOC_MIRROR` +and `R_BIOC_VERSION` environment variables below to configure pkgcache's +Bioconductor support. + +## Package Options + +- The `BioC_mirror` option can be used to select a Bioconductor mirror. + This takes priority over the `R_BIOC_MIRROR` environment variable. +- `pkgcache_timeout` is the HTTP timeout for all downloads. It is in + seconds, and the limit for downloading the whole file. Defaults to 3600, + one hour. It corresponds to the [`TIMEOUT` libcurl option](https://curl.haxx.se/libcurl/c/CURLOPT_TIMEOUT.html). +- `pkgcache_connecttimeout` is the HTTP timeout for the connection phase. + It is in seconds and defaults to 30 seconds. It corresponds to the + [`CONNECTTIMEOUT` libcurl option](https://curl.haxx.se/libcurl/c/CURLOPT_CONNECTTIMEOUT.html). +- `pkgcache_low_speed_limit` and `pkgcache_low_speed_time` are used for a + more sensible HTTP timeout. If the download speed is less than + `pkgcache_low_speed_limit` bytes per second for at least + `pkgcache_low_speed_time` seconds, the download errors. They correspond + to the [`LOW_SPEED_LIMIT`](https://curl.haxx.se/libcurl/c/CURLOPT_LOW_SPEED_LIMIT.html) and + [`LOW_SPEED_TIME`](https://curl.haxx.se/libcurl/c/CURLOPT_LOW_SPEED_TIME.html) + curl options. + +## Package environment variables + +- The `R_BIOC_VERSION` environment variable can be used to override the + default Bioconductor version detection and force a given version. E.g. + this can be used to force the development version of Bioconductor. +- The `R_BIOC_MIRROR` environment variable can be used to select a + Bioconductor mirror. The `BioC_mirror` option takes priority over this, + if set. +- `PKGCACHE_TIMEOUT` is the HTTP timeout for all downloads. It is in + seconds, and the limit for downloading the whole file. Defaults to 3600, + one hour. It corresponds to the [`TIMEOUT` libcurl option](https://curl.haxx.se/libcurl/c/CURLOPT_TIMEOUT.html). + The `pkgcache_timeout` option has priority over this, if set. +- `PKGCACHE_CONNECTTIMEOUT` is the HTTP timeout for the connection phase. + It is in seconds and defaults to 30 seconds. It corresponds to the + [`CONNECTTIMEOUT` libcurl option](https://curl.haxx.se/libcurl/c/CURLOPT_CONNECTTIMEOUT.html). The + `pkgcache_connecttimeout` option takes precedence over this, if set. +- `PKGCACHE_LOW_SPEED_LIMIT` and `PKGCACHE_LOW_SPEED_TIME` are used for a + more sensible HTTP timeout. If the download speed is less than + `PKGCACHE_LOW_SPEED_LIMIT` bytes per second for at least + `PKGCACHE_LOW_SPEED_TIME` seconds, the download errors. They correspond + to the [`LOW_SPEED_LIMIT`](https://curl.haxx.se/libcurl/c/CURLOPT_LOW_SPEED_LIMIT.html) and + [`LOW_SPEED_TIME`](https://curl.haxx.se/libcurl/c/CURLOPT_LOW_SPEED_TIME.html) + curl options. The `pkgcache_low_speed_time` and `pkgcache_low_speed_limit` + options have priority over these environment variables, if they are set. + +## Code of Conduct + +Please note that the 'pkgcache' project is released with a +[Contributor Code of Conduct](https://github.com/r-lib/pkgcache/blob/master/.github/CODE_OF_CONDUCT.md). +By contributing to this project, you agree to abide by its terms. + +## License + +MIT © [RStudio Inc](https://rstudio.com) From 2eb1f754762b13b0dc48f1bc72326c5c2df9cedb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Cs=C3=A1rdi?= Date: Thu, 12 Dec 2019 11:46:26 +0000 Subject: [PATCH 8/9] No Unicode in fixed width fonts in the manual :( --- man/pkgcache-package.Rd | 4 ++-- tools/README-body.Rmd | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/man/pkgcache-package.Rd b/man/pkgcache-package.Rd index d9d2bb76..0f25a473 100644 --- a/man/pkgcache-package.Rd +++ b/man/pkgcache-package.Rd @@ -28,8 +28,8 @@ includes Bioconductor package, and all versions (i.e. both binary and source) of the packages for the current platform and R version.\if{html}{\out{
}}\preformatted{library(pkgcache) meta_cache_list() #> -ℹ Loading global cached package metadata -✔ Loading global cached package metadata +i Loading global cached package metadata +v Loading global cached package metadata #> # A tibble: 35,272 x 33 #> package title version depends suggests built imports archs repodir #> diff --git a/tools/README-body.Rmd b/tools/README-body.Rmd index 66a738a6..c80b3d48 100644 --- a/tools/README-body.Rmd +++ b/tools/README-body.Rmd @@ -5,6 +5,7 @@ knitr::opts_chunk$set( comment = "#>", cache = TRUE ) +options(cli.unicode = FALSE) ``` Metadata and package cache for CRAN-like repositories. This is a utility From 96057e3c08a628c0c7bf0df7a65e267996795c1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Cs=C3=A1rdi?= Date: Thu, 12 Dec 2019 12:00:34 +0000 Subject: [PATCH 9/9] Add NEWS for new HTTP timeouts [ci skip] --- NEWS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS.md b/NEWS.md index 0caac036..9aeef61b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,9 @@ # pkgcache dev +* HTTP timeouts are now much better, and by default they are defined + in terms of download speed, instead of total download time (#29). + * pkgcache now tries to download metadata from the `PACKAGES` file, if it cannot find `PACKAGES.gz` (@timmsm, #27).