Use downlit (#1348)

downlit is code (and tests) extracted out from pkgdown so this shouldn't cause any user facing changes assuming that I've plumbed together all the options correctly.

Fixes #1234

DESCRIPTION

@@ -24,13 +24,12 @@ Imports:
     crayon,
     desc,
     digest,
+    downlit,
     evaluate,
     fs (>= 1.3.0),
     fansi,
-    highlight,
     httr,
     magrittr,
-    MASS,
     memoise,
     openssl,
     purrr,
@@ -55,6 +54,7 @@ Suggests:
     testthat (>= 2.1.0),
     rticles,
     rsconnect
+Remotes: r-lib/downlit, rstudio/rmarkdown
 VignetteBuilder: knitr
 SystemRequirements: pandoc
 RoxygenNote: 7.1.0

NAMESPACE

@@ -111,7 +111,6 @@ S3method(usage_code,tag_dots)
 S3method(usage_code,tag_ldots)
 S3method(usage_code,tag_method)
 S3method(usage_code,tag_usage)
-export(addterm)
 export(as_pkgdown)
 export(autolink_html)
 export(build_article)
@@ -139,7 +138,6 @@ export(template_navbar)
 export(template_reference)
 import(fs)
 import(rlang)
-importFrom(MASS,addterm)
 importFrom(magrittr,"%>%")
 importFrom(memoise,memoise)
 importFrom(utils,installed.packages)

NEWS.md

@@ -1,5 +1,14 @@
 # pkgdown (development version)
 
+* pkgdown now uses the new downlit package for all syntax highlighting and 
+  autolinking (in both reference topics and vignettes). There should be very
+  little change in behaviour because the code in downlit was extracted from
+  pkgdown, but this makes it easier to use pkgdown's nice linking/highlighting
+  in more places (#1234).
+
+* `autolink_html()` is (soft) deprecated. Please use 
+  `downlit::downlit_html_path()` instead.
+
 * Suppressing CRAN dates in news file now actualy works.
 
 * All HTTP requests are now retried upon failure (@jameslamb, #1305).

R/autolink_html.R

@@ -1,35 +1,13 @@
 #' Automatically link references and articles in an HTML page
 #'
-#' @description
-#' The autolinker is built around two XPath expressions:
-#'
-#' * `//pre[contains(@class, 'r')]`:
-#'   this finds all `<div>`s with class `sourceCode` and `r`. The contents
-#'   must be syntax-highlighted using [pygments](http://pygments.org/).
-#'   (This is default in [rmarkdown::html_document()] when `theme = NULL`.)
-#'
-#' * `.//code[count(*) = 0]`: this finds all `<code>` that contain only
-#'   text (and no other tags).
-#'
-#' @details
-#' Currently the following expressions are linked:
-#'
-#' * Function calls, `foo()`
-#' * Function calls qualified with the package name, `bar::foo()`
-#' * Symbols qualified with the package name, `bar::baz`
-#' * `library()`, `require()` and `requireNamespace()` calls.
-#' * Help calls, `?foo`, `package?foo`, `?bar::foo`, `help(foo)`,
-#'   `help(foo, package = bar)`, `help(package = bar)`.
-#' * Vignette calls, `vignette(baz)`, `vignette(baz, package = "bar")`
-#'
-#' Calls to `library()` and `require()` are used to find the topics connected
-#' to unqualified references.
+#' Deprecated: please use [downlit::downlit_html_path] instead.
 #'
 #' @param input,output Input and output paths for HTML file
 #' @param local_packages A named character vector providing relative paths
 #'   (value) to packages (name) that can be reached with relative links
 #'   from the target HTML document.
 #' @export
+#' @keywords internal
 #' @examples
 #' \dontrun{
 #' autolink_html("path/to/file.html",
@@ -40,16 +18,13 @@
 #' )
 #' }
 autolink_html <- function(input, output = input, local_packages = character()) {
-  scoped_package_context(
-    package = "",
-    topic_index = character(),
-    article_index = character(),
-    local_packages = local_packages
-  )
-  scoped_file_context()
+  withr::local_options(list(
+    downlit.package = "",
+    downlit.local_packages = local_packages
+  ))
 
   html <- xml2::read_html(input, encoding = "UTF-8")
-  tweak_code(html)
+  downlit::downlit_html_node(html)
 
   xml2::write_html(html, output, format = FALSE)
   invisible()

R/build-articles.R

@@ -234,9 +234,8 @@ build_article <- function(name,
     return(invisible())
   }
 
-  scoped_in_pkgdown()
-  scoped_package_context(pkg$package, pkg$topic_index, pkg$article_index)
-  scoped_file_context(depth = depth)
+  local_envvar_pkgdown()
+  local_options_link(pkg, depth = depth)
 
   front <- rmarkdown::yaml_front_matter(input_path)
   # Take opengraph from article's yaml front matter

R/build-home-index.R

@@ -1,9 +1,6 @@
 build_home_index <- function(pkg = ".", quiet = TRUE) {
   pkg <- as_pkgdown(pkg)
 
-  scoped_package_context(pkg$package, pkg$topic_index, pkg$article_index)
-  scoped_file_context(depth = 0L)
-
   src_path <- path_first_existing(
     pkg$src_path,
     c("pkgdown/index.md",
@@ -17,6 +14,7 @@ build_home_index <- function(pkg = ".", quiet = TRUE) {
   if (is.null(src_path)) {
     data$index <- linkify(pkg$desc$get("Description")[[1]])
   } else {
+    local_options_link(pkg, depth = 0L)
     data$index <- markdown(src_path)
   }
   render_page(pkg, "home", data, "index.html", quiet = quiet)
@@ -85,6 +83,9 @@ sidebar_section <- function(heading, bullets, class = make_slug(heading)) {
   )
 }
 
+#' @importFrom memoise memoise
+NULL
+
 cran_link <- memoise(function(pkg) {
   if (!has_internet()) {
     return(NULL)

R/build-news.R

@@ -132,12 +132,12 @@ globalVariables(".")
 
 data_news <- function(pkg = ".") {
   pkg <- as_pkgdown(pkg)
-  scoped_file_context(depth = 1L)
 
   html <- markdown(path(pkg$src_path, "NEWS.md"))
+  xml <- xml2::read_html(html)
+  downlit::downlit_html_node(xml)
 
-  sections <- xml2::read_html(html) %>%
-    xml2::xml_find_all("./body/div")
+  sections <- xml2::xml_find_all(xml, "./body/div")
 
   titles <- sections %>%
     xml2::xml_find_first(".//h1|h2") %>%
@@ -161,7 +161,6 @@ data_news <- function(pkg = ".") {
   }
 
   html <- sections %>%
-    purrr::walk(tweak_code) %>%
     purrr::walk2(versions, tweak_news_heading, timeline = timeline) %>%
     purrr::map_chr(as.character) %>%
     purrr::map_chr(repo_auto_link, pkg = pkg)

R/build-reference.R

@@ -230,14 +230,23 @@ build_reference_topic <- function(topic,
     return(invisible())
 
   cat_line("Reading ", src_path("man", topic$file_in))
-  scoped_file_context(rdname = path_ext_remove(topic$file_in), depth = 1L)
 
-  data <- data_reference_topic(
-    topic,
-    pkg,
-    examples = examples,
-    run_dont_run = run_dont_run
+  data <- withCallingHandlers(
+    data_reference_topic(
+      topic,
+      pkg,
+      examples = examples,
+      run_dont_run = run_dont_run
+    ),
+    error = function(err) {
+      msg <- c(
+        paste0("Failed to parse Rd in ", topic$file_in),
+        i = err$message
+      )
+      abort(msg, parent = err)
+    }
   )
+
   render_page(
     pkg, "reference-topic",
     data = data,
@@ -254,6 +263,9 @@ data_reference_topic <- function(topic,
                                  examples = TRUE,
                                  run_dont_run = FALSE
                                  ) {
+  local_context_eval(pkg$figures, pkg$src_path)
+  withr::local_options(list(downlit.rdname = topic$name))
+
   tag_names <- purrr::map_chr(topic$rd, ~ class(.)[[1]])
   tags <- split(topic$rd, tag_names)
 

R/context.R

@@ -1,20 +1,37 @@
-section_init <- function(pkg, depth, override = list(), scope = parent.frame()) {
+section_init <- function(pkg, depth, override = list(), .frame = parent.frame()) {
   pkg <- as_pkgdown(pkg, override = override)
 
   rstudio_save_all()
-  scoped_in_pkgdown(scope = scope)
+  local_envvar_pkgdown()
+  local_options_link(pkg, depth = depth)
 
-  scoped_package_context(
-    package = pkg$package,
-    topic_index = pkg$topic_index,
-    article_index = pkg$article_index,
-    figures = pkg$figures,
-    src_path = pkg$src_path,
-    scope = scope
+  pkg
+}
+
+local_options_link <- function(pkg, depth, .frame = parent.frame()) {
+  article_index <- set_names(path_file(pkg$vignettes$file_out), pkg$vignettes$name)
+  topic_index <- invert_index(set_names(pkg$topics$alias, pkg$topics$name))
+
+  withr::local_options(
+    list(
+      downlit.package = pkg$package,
+      downlit.article_index = article_index,
+      downlit.topic_index = topic_index,
+      downlit.article_path = paste0(up_path(depth), "articles/"),
+      downlit.topic_path = paste0(up_path(depth), "reference/")
+    ),
+    .local_envir = .frame
   )
-  scoped_file_context(depth = depth, scope = scope)
+}
 
-  pkg
+local_context_eval <- function(
+                               figures = NULL,
+                               src_path = getwd(),
+                               sexpr_env = child_env(globalenv()),
+                               .frame = parent.frame()) {
+  context_set_scoped("figures", figures, scope = .frame)
+  context_set_scoped("src_path", src_path, scope = .frame)
+  context_set_scoped("sexpr_env", sexpr_env, scope = .frame)
 }
 
 # Manage current topic index ----------------------------------------------------
@@ -45,63 +62,5 @@ context_get <- function(name) {
 
 context_set_scoped <- function(name, value, scope = parent.frame()) {
   old <- context_set(name, value)
-  defer(context_set(name, old), scope = scope)
-}
-
-scoped_package_context <- function(package,
-                                   topic_index = NULL,
-                                   article_index = NULL,
-                                   local_packages = character(),
-                                   src_path = getwd(),
-                                   figures = list(),
-                                   scope = parent.frame()) {
-  stopifnot(is.character(local_packages))
-
-  topic_index <- topic_index %||% topic_index(package)
-  article_index <- article_index %||% article_index(package)
-
-  context_set_scoped("package", package, scope = scope)
-  context_set_scoped("topic_index", topic_index, scope = scope)
-  context_set_scoped("article_index", article_index, scope = scope)
-  context_set_scoped("local_packages", local_packages, scope = scope)
-  context_set_scoped("figures", figures, scope = scope)
-  context_set_scoped("src_path", src_path, scope = scope)
-
-
-}
-scoped_file_context <- function(rdname = "",
-                                depth = 0L,
-                                packages = character(),
-                                scope = parent.frame(),
-                                sexpr_env = child_env(globalenv())) {
-
-  # Base packages are always attached
-  packages <- union(
-    packages,
-    c("base", "stats", "graphics", "grDevices", "utils", "datasets")
-  )
-
-  context_set_scoped("rdname", rdname, scope = scope)
-  context_set_scoped("depth", depth, scope = scope)
-  context_set_scoped("packages", packages, scope = scope)
-  context_set_scoped("sexpr_env", sexpr_env, scope = scope)
-}
-
-# Unlike file and package contexts, the attached context can be
-# built up over multiple calls, as we encounter new calls to
-# library() or require()
-register_attached_packages <- function(packages) {
-  packages <- union(packages, context_get("packages"))
-  context_set("packages", packages)
-}
-
-# defer helper ------------------------------------------------------------
-
-defer <- function(expr, scope = parent.frame()) {
-  expr <- enquo(expr)
-
-  call <- expr(on.exit(rlang::eval_tidy(!!expr), add = TRUE))
-  eval_bare(call, scope)
-
-  invisible()
+  withr::defer(context_set(name, old), envir = scope)
 }