Merge branch 'master' into symbols-docs

.Rbuildignore

@@ -20,7 +20,6 @@
 
 ^Makefile$
 ^NEWS\.0\.md$
-^README\.md$
 ^_pkgdown\.yml$
 ^src/Makevars$
 

.appveyor.yml

@@ -32,7 +32,7 @@ environment:
 
   - R_VERSION: release  # the single Windows.zip binary (both 32bit/64bit) that users following dev version of installation instructions should click
 
-  - R_VERSION: devel   # When off it's to speed up dev cycle; R-devel is still checked but by GLCI on a roughly hourly cycle. CRAN_Release.cmd has a reminder to turn back on.
+#  - R_VERSION: devel   # Never turn back on. GLCI after merge covers latest daily R-devel very well, so we shouldn't confuse and slow down PR dev cycle by measuring PRs against daily R-devel too. If a change in R-devel yesterday breaks the PR, it's very unlikely to be due to something in the PR. So we should accept the PR if it passes R-release and fix separately anything related to R-devel which we'll see from GLCI.
 
 before_build:
   - cmd: ECHO no Revision metadata added to DESCRIPTION

.ci/README.md

@@ -16,6 +16,7 @@ Test jobs:
 - `test-350-cran-lin` - R 3.5.0 on Linux, no `r-recommended`
 - `test-rel-win` - `r-release` on Windows
 - `test-dev-win` - `r-devel` on Windows
+- `test-old-win` - `r-oldrel` on Windows
 - `test-rel-osx` - MacOSX build not yet deployed, see [#3326](https://github.com/Rdatatable/data.table/issues/3326) for status
 
 Artifacts:
@@ -25,7 +26,7 @@ Artifacts:
 - [html vignettes](https://rdatatable.gitlab.io/data.table/library/data.table/doc/index.html)
 - R packages repository for `data.table` and all _Suggests_ dependencies, url: `https://Rdatatable.gitlab.io/data.table`
   - sources
-  - Windows binaries for `r-release` and `r-devel`
+  - Windows binaries for `r-release`, `r-devel` and `r-oldrel`
 - [CRAN-like homepage](https://rdatatable.gitlab.io/data.table/web/packages/data.table/index.html)
 - [CRAN-like checks results](https://rdatatable.gitlab.io/data.table/web/checks/check_results_data.table.html) - note that all artifacts, including check results page, are being published only when all test jobs successfully pass, thus one will not see an _ERROR_ status there (unless error happened on a job marked as `allow_failure`).
 - [docker images](https://gitlab.com/Rdatatable/data.table/container_registry) - copy/paste-able `docker pull` commands can be found at the bottom of our [CRAN-like homepage](https://rdatatable.gitlab.io/data.table/web/packages/data.table/index.html)

.ci/publish.R

@@ -1,12 +1,17 @@
 format.deps <- function(file, which) {
   deps.raw = read.dcf(file, fields=which)[[1L]]
   if (all(is.na(deps.raw))) return(character())
+  deps.raw = gsub("\n", " ", deps.raw, fixed=TRUE)
   deps.full = trimws(strsplit(deps.raw, ", ", fixed=TRUE)[[1L]])
   deps = trimws(sapply(strsplit(deps.full, "(", fixed=TRUE), `[[`, 1L))
+  deps.full = gsub(">=", "&ge;", deps.full, fixed=TRUE)
+  deps.full = gsub("<=", "&le;", deps.full, fixed=TRUE)
+  if (any(grepl(">", deps.full, fixed=TRUE), grepl("<", deps.full, fixed=TRUE), grepl("=", deps.full, fixed=TRUE)))
+    stop("formatting dependencies version for CRAN-line package website failed because some dependencies have version defined using operators other than >= and <=")
   names(deps.full) <- deps
   base.deps = c("R", unlist(tools:::.get_standard_package_names(), use.names = FALSE))
   ans = sapply(deps, function(x) {
-    if (x %in% base.deps) deps.full[[x]]
+    if (x %in% base.deps) deps.full[[x]] ## base R packages are not linked
     else sprintf("<a href=\"../%s/index.html\">%s</a>", x, deps.full[[x]])
   })
   sprintf("<tr><td>%s:</td><td>%s</td></tr>", which, paste(ans, collapse=", "))
@@ -26,6 +31,39 @@ format.bins <- function(ver, bin_ver, cran.home, os.type, pkg, version, repodir)
   paste(ans[fe], collapse=", ")
 }
 
+format.entry <- function(field, dcf, url=FALSE) {
+  if (field %in% colnames(dcf)) {
+    value = gsub("\n", " ", dcf[,field], fixed=TRUE)
+    if (url) {
+      urls = trimws(strsplit(value, ",", fixed=TRUE)[[1L]])
+      value = paste(sprintf("<a href=\"%s\">%s</a>", urls, urls), collapse=", ")
+    }
+    sprintf("<tr><td>%s:</td><td>%s</td></tr>", field, value)
+  }
+}
+format.maintainer <- function(dcf) {
+  if ("Maintainer" %in% colnames(dcf)) {
+    text2html = function(x) {
+      # https://stackoverflow.com/a/64446320/2490497
+      splitted <- strsplit(x, "")[[1L]]
+      intvalues <- as.hexmode(utf8ToInt(enc2utf8(x)))
+      paste(paste0("&#x", intvalues, ";"), collapse = "")
+    }
+    tmp = gsub("@", " at ", dcf[,"Maintainer"], fixed=TRUE)
+    sep = regexpr("<", tmp, fixed=TRUE)
+    name = trimws(substr(tmp, 1L, sep-1L))
+    mail = text2html(trimws(substr(tmp, sep, nchar(tmp))))
+    sprintf("<tr><td>Maintainer:</td><td>%s  %s</td></tr>", name, mail)
+  }
+}
+format.materials <- function() {
+  return(NULL) ## TODO
+  value = NA
+  #NEWS
+  #README
+  sprintf("<tr><td>Materials:</td><td>%s</td></tr>", value)
+}
+
 package.index <- function(package, lib.loc, repodir="bus/integration/cran") {
   file = system.file("DESCRIPTION", package=package, lib.loc=lib.loc)
   dcf = read.dcf(file)
@@ -40,21 +78,31 @@ package.index <- function(package, lib.loc, repodir="bus/integration/cran") {
     format.deps(file, "LinkingTo"),
     format.deps(file, "Suggests"),
     format.deps(file, "Enhances"),
+    if ("Built" %in% colnames(dcf)) sprintf("<tr><td>Built:</td><td>%s</td></tr>", substr(trimws(strsplit(dcf[,"Built"], ";", fixed=TRUE)[[1L]][[3L]]), 1L, 10L)),
+    if ("Author" %in% colnames(dcf)) sprintf("<tr><td>Author:</td><td>%s</td></tr>", dcf[,"Author"]),
+    format.maintainer(dcf),
+    format.entry("BugReports", dcf, url=TRUE),
+    format.entry("License", dcf),
+    format.entry("URL", dcf, url=TRUE),
+    format.entry("NeedsCompilation", dcf),
+    format.entry("SystemRequirements", dcf),
+    format.materials(), ## TODO
     if (pkg=="data.table") sprintf("<tr><td>Checks:</td><td><a href=\"../../checks/check_results_%s.html\">%s results</a></td></tr>", pkg, pkg)
   )
   vign = tools::getVignetteInfo(pkg, lib.loc=lib.loc)
-  r_bin_ver = Sys.getenv("R_BIN_VERSION")
-  r_devel_bin_ver = Sys.getenv("R_DEVEL_BIN_VERSION")
-  stopifnot(nzchar(r_bin_ver), nzchar(r_devel_bin_ver))
+  r_rel_ver = Sys.getenv("R_REL_VERSION")
+  r_devel_ver = Sys.getenv("R_DEVEL_VERSION")
+  r_oldrel_ver = Sys.getenv("R_OLDREL_VERSION")
+  stopifnot(nzchar(r_rel_ver), nzchar(r_devel_ver), nzchar(r_oldrel_ver))
   cran.home = "../../.."
   tbl.dl = c(
     sprintf("<tr><td> Reference manual: </td><td> <a href=\"%s.pdf\">%s.pdf</a>, <a href=\"%s/library/%s/html/00Index.html\">00Index.html</a> </td></tr>", pkg, pkg, cran.home, pkg),
     if (nrow(vign)) sprintf("<tr><td>Vignettes:</td><td>%s</td></tr>", paste(sprintf("<a href=\"%s/library/data.table/doc/%s\">%s</a><br/>", cran.home, vign[,"PDF"], vign[,"Title"]), collapse="\n")), # location unline cran web/pkg/vignettes to not duplicate content, documentation is in ../../../library
     sprintf("<tr><td> Package source: </td><td> <a href=\"%s/src/contrib/%s_%s.tar.gz\"> %s_%s.tar.gz </a> </td></tr>", cran.home,pkg, version, pkg, version),
-    sprintf("<tr><td> Windows binaries: </td><td> %s </td></tr>", format.bins(ver=c("r-devel","r-release"), bin_ver=c(r_devel_bin_ver,r_bin_ver), cran.home=cran.home, os.type="windows", pkg=pkg, version=version, repodir=repodir)),
-    sprintf("<tr><td> OS X binaries: </td><td> %s </td></tr>", format.bins(ver=c("r-devel","r-release"), bin_ver=c(r_devel_bin_ver, r_bin_ver), cran.home=cran.home, os.type="macosx", pkg=pkg, version=version, repodir=repodir))
+    sprintf("<tr><td> Windows binaries: </td><td> %s </td></tr>", format.bins(ver=c("r-devel","r-release","r-oldrel"), bin_ver=c(r_devel_ver, r_rel_ver, r_oldrel_ver), cran.home=cran.home, os.type="windows", pkg=pkg, version=version, repodir=repodir)),
+    sprintf("<tr><td> macOS binaries: </td><td> %s </td></tr>", format.bins(ver=c("r-release","r-oldrel"), bin_ver=c(r_rel_ver, r_oldrel_ver), cran.home=cran.home, os.type="macosx", pkg=pkg, version=version, repodir=repodir))
   )
-  if (pkg=="data.table") {
+  if (pkg=="data.table") { ## docker images
     registry = Sys.getenv("CI_REGISTRY", "registry.gitlab.com")
     namespace = Sys.getenv("CI_PROJECT_NAMESPACE", "Rdatatable")
     project = Sys.getenv("CI_PROJECT_NAME", "data.table")
@@ -74,7 +122,7 @@ package.index <- function(package, lib.loc, repodir="bus/integration/cran") {
     "<style type=\"text/css\">table td { vertical-align: top; }</style>",
     "</head>",
     "<body>",
-    sprintf("<h2>%s</h2>", dcf[,"Title"]),
+    sprintf("<h2>%s: %s</h2>", pkg, dcf[,"Title"]),
     sprintf("<p>%s</p>", dcf[,"Description"]),
     sprintf("<table summary=\"Package %s summary\">", pkg),
     tbl,
@@ -117,7 +165,48 @@ doc.copy <- function(repodir="bus/integration/cran"){
   c(ans1, ans2)
 }
 
-plat <- function(x) if (grepl("^.*win", x)) "Windows" else if (grepl("^.*osx", x)) "Mac OS X" else "Linux"
+plat <- function(x) if (grepl("^.*win", x)) "Windows" else if (grepl("^.*mac", x)) "macOS" else "Linux"
+
+r.ver <- function(x) {
+  tmp = strsplit(x, "-", fixed=TRUE)[[1L]]
+  if (length(tmp) < 2L) stop("test job names must be test-[r.version]-...")
+  v = tmp[2L]
+  if (identical(v, "rel")) "r-release"
+  else if (identical(v, "dev")) "r-devel"
+  else if (identical(v, "old")) "r-oldrel"
+  else {
+    if (grepl("\\D", v)) stop("second word in test job name must be rel/dev/old or numbers of R version")
+    paste0("r-", paste(strsplit(v, "")[[1L]], collapse="."))
+  }
+}
+
+# this for now is constant but when we move to independent pipelines (commit, daily, weekly) those values can be different
+pkg.version <- function(job, pkg) {
+  dcf = read.dcf(file.path("bus", job, paste(pkg, "Rcheck", sep="."), pkg, "DESCRIPTION"))
+  dcf[,"Version"]
+}
+pkg.revision <- function(job, pkg) {
+  dcf = read.dcf(file.path("bus", job, paste(pkg, "Rcheck", sep="."), pkg, "DESCRIPTION"))
+  if ("Revision" %in% colnames(dcf)) {
+    proj.url = Sys.getenv("CI_PROJECT_URL", "")
+    if (!nzchar(proj.url)) {
+      warning("pkg.revision was designed to be run on GLCI where CI_PROJECT_URL var is set, links to commits will not be produced for checks table")
+      substr(dcf[,"Revision"], 1, 7)
+    } else {
+      sprintf("<a href=\"%s\">%s</a>", file.path(proj.url, "-", "commit", dcf[,"Revision"]), substr(dcf[,"Revision"], 1, 7))
+    }
+  } else ""
+}
+pkg.flags <- function(job, pkg) {
+  cc = file.path("bus", job, paste(pkg, "Rcheck", sep="."), pkg, "cc") ## data.table style cc file
+  if (file.exists(cc)) {
+    d = readLines(cc)
+    w.cflags = substr(d, 1, 7)=="CFLAGS="
+    if (sum(w.cflags)==1L)
+      return(sub("CFLAGS=", "", d[w.cflags], fixed=TRUE))
+  }
+  ""
+}
 
 check.copy <- function(job, repodir="bus/integration/cran"){
   dir.create(job.checks<-file.path(repodir, "web", "checks", pkg<-"data.table", job), recursive=TRUE);
@@ -146,6 +235,39 @@ check.copy <- function(job, repodir="bus/integration/cran"){
   setNames(file.exists(file.path(job.checks, c(inst.check, routs))), c(inst.check, routs))
 }
 
+check.flavors <- function(jobs, repodir="bus/integration/cran") {
+  th = "<th>Flavor</th><th>R Version</th><th>OS Type</th><th>CPU Type</th><th>OS Info</th><th>CPU Info</th><th>Compilers</th>"
+  tbl = sprintf(
+    "<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>",
+    sub("test-", "", jobs, fixed=TRUE),
+    sapply(jobs, r.ver),
+    sapply(jobs, plat),
+    "", # "x86_64"
+    "", # "Debian GNU/Linux testing"
+    "", # "2x 8-core Intel(R) Xeon(R) CPU E5-2690 0 @ 2.90GHz"
+    "" # "GCC 10.2.0 (Debian 10.2.0-13)"
+  )
+  file = file.path(repodir, "web/checks", "check_flavors.html")
+  writeLines(c(
+    "<html>",
+    "<head>",
+    "<title>Package Check Flavors</title>",
+    "<link rel=\"stylesheet\" type=\"text/css\" href=\"../CRAN_web.css\"/>",
+    "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"/>",
+    "</head>",
+    "<body lang=\"en\">",
+    "<h2>Package Check Flavors</h2>",
+    sprintf("<p>Last updated on %s.</p>", format(Sys.time(), usetz=TRUE)),
+    "<table border=\"1\" summary=\"CRAN check flavors.\">",
+    "<tr>",th,"</tr>",
+    tbl,
+    "</table>",
+    "</body>",
+    "</html>"
+  ), file)
+  setNames(file.exists(file), file)
+}
+
 check.index <- function(pkg, jobs, repodir="bus/integration/cran") {
   status = function(x) if (grepl("^.*ERROR", x)) "ERROR" else if (grepl("^.*WARNING", x)) "WARNING" else if (grepl("^.*NOTE", x)) "NOTE" else if (grepl("^.*OK", x)) "OK" else NA_character_
   test.files = function(job, files, trim.name=FALSE, trim.exts=0L, pkg="data.table") {
@@ -186,30 +308,36 @@ check.index <- function(pkg, jobs, repodir="bus/integration/cran") {
     }
     memouts
   })
-  tbl = sprintf("<tr><td>%s</td><td>%s</td><td><a href=\"%s/%s/00install.out\">out</a></td><td><a href=\"%s/%s/00check.log\">%s</a></td><td>%s</td><td>%s</td></tr>",
-                sub("test-", "", jobs, fixed=TRUE),
-                sapply(jobs, plat),
-                pkg, jobs,
-                pkg, jobs, sapply(sapply(jobs, check.test, pkg="data.table"), status),
-                mapply(test.files, jobs, routs, trim.exts=2L), # 1st fail, 2nd Rout, keep just: tests_x64/main
-                mapply(test.files, jobs, memouts, trim.name=TRUE))
+  th = "<th>Flavor</th><th>Version</th><th>Revision</th><th>Install</th><th>Status</th><th>Flags</th><th>Rout.fail</th><th>Memtest</th>"
+  tbl = sprintf(
+    "<tr><td><a href=\"check_flavors.html\">%s</a></td><td>%s</td><td>%s</td><td><a href=\"%s/%s/00install.out\">out</a></td><td><a href=\"%s/%s/00check.log\">%s</a></td><td>%s</td><td>%s</td><td>%s</td></tr>",
+    sub("test-", "", jobs, fixed=TRUE),
+    sapply(jobs, pkg.version, pkg),
+    sapply(jobs, pkg.revision, pkg),
+    pkg, jobs, ## install
+    pkg, jobs, sapply(sapply(jobs, check.test, pkg="data.table"), status), ## check
+    sapply(jobs, pkg.flags, pkg),
+    mapply(test.files, jobs, routs, trim.exts=2L), # 1st fail, 2nd Rout, keep just: tests_x64/main
+    mapply(test.files, jobs, memouts, trim.name=TRUE)
+  )
   file = file.path(repodir, "web/checks", sprintf("check_results_%s.html", pkg))
-  writeLines(c("<html>",
-               "<head>",
-               sprintf("<title>Package Check Results for Package %s</title>", pkg),
-               "<link rel=\"stylesheet\" type=\"text/css\" href=\"../CRAN_web.css\"/>",
-               "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"/>",
-               "</head>",
-               "<body lang=\"en\">",
-               sprintf("<h2>Package Check Results for Package <a href=\"../packages/%s/index.html\"> %s </a> </h2>", pkg, pkg),
-               sprintf("<p>Last updated on %s.</p>", format(Sys.time(), usetz=TRUE)),
-               sprintf("<table border=\"1\" summary=\"CRAN check results for package %s\">", pkg),
-               "<tr><th>Test job</th><th>OS type</th><th>Install</th><th>Check</th><th>Rout.fail</th><th>Memtest</th></tr>",
-               tbl,
-               "</table>",
-               "</body>",
-               "</html>"),
-             file)
+  writeLines(c(
+    "<html>",
+    "<head>",
+    sprintf("<title>Package Check Results for Package %s</title>", pkg),
+    "<link rel=\"stylesheet\" type=\"text/css\" href=\"../CRAN_web.css\"/>",
+    "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"/>",
+    "</head>",
+    "<body lang=\"en\">",
+    sprintf("<h2>Package Check Results for Package <a href=\"../packages/%s/index.html\"> %s </a> </h2>", pkg, pkg),
+    sprintf("<p>Last updated on %s.</p>", format(Sys.time(), usetz=TRUE)),
+    sprintf("<table border=\"1\" summary=\"CRAN check results for package %s\">", pkg),
+    "<tr>",th,"</tr>",
+    tbl,
+    "</table>",
+    "</body>",
+    "</html>"
+  ), file)
   setNames(file.exists(file), file)
 }
 

.dev/.bash_aliases

@@ -5,7 +5,7 @@
 # git config --global difftool.prompt false
 alias gd='git difftool &> /dev/null'
 alias gdm='git difftool master &> /dev/null'
-# If meld has scrolling issues, turn off GTK animation (which I don't need anyway):
+# If meld has scrolling issues, turn off GTK animation which I don't need:
 #   https://gitlab.gnome.org/GNOME/meld/-/issues/479#note_866040
 
 alias Rdevel='~/build/R-devel/bin/R --vanilla'
@@ -14,11 +14,20 @@ alias Rdevel-strict-clang='~/build/R-devel-strict-clang/bin/R --vanilla'
 alias Rdevel-valgrind='~/build/R-devel-valgrind/bin/R --vanilla'
 alias Rdevel32='~/build/32bit/R-devel/bin/R --vanilla'
 alias R310='~/build/R-3.1.0/bin/R --vanilla'
-alias revdepsh='cd ~/build/revdeplib/ && export TZ=UTC && export R_LIBS_SITE=none && export R_LIBS=~/build/revdeplib/ && export _R_CHECK_FORCE_SUGGESTS_=false'
-alias revdepr='revdepsh; R_PROFILE_USER=~/GitHub/data.table/.dev/revdep.R ~/build/R-devel/bin/R'
+
+alias revdepsh='cd ~/build/revdeplib/ && export TZ=UTC && export R_LIBS_SITE=none && export R_LIBS=~/build/revdeplib/ && export _R_CHECK_FORCE_SUGGESTS_=true'
+alias revdepr='revdepsh; R_PROFILE_USER=~/GitHub/data.table/.dev/revdep.R R'
+# use ~/build/R-devel/bin/R at the end of revdepr to use R-devel instead of R-release.
+# If so, doing a `rm -rf *` in revdeplib first to rebuild everything is easiest way to avoid potential problems later. A full rebuild is a good idea periodically anyway. Packages in
+# revdeplib may have been compiled many months ago, but the .so libraries they link to may have been updated in the meantime, or multiple packages may use the same .so libary, or
+# switches inside the package's code may behave differently when R-devel is used instead of R-release, etc. I use R-release for revdepr, unless R-devel contains significant changes
+# that we really need to test revdeps under.
 
 export R_PROFILE_USER='~/.Rprofile'
 # there's a .Rprofile in ~/GitHub/data.table/ so Matt sets R_PROFILE_USER here to always use ~/.Rprofile
 # even when starting R in ~/GitHub/data.table
 # Matt's ~/.Rprofile as a link to ~/GitHub/data.table/.dev/.Rprofile
 
+export R_DEFAULT_INTERNET_TIMEOUT=3600
+# increase from R's default 60, always not just in revdep testing, to help --as-cran
+