CI improvements (#4768)

.ci/README.md

@@ -16,6 +16,7 @@ Test jobs:
 - `test-350-cran-lin` - R 3.5.0 on Linux, no `r-recommended`
 - `test-rel-win` - `r-release` on Windows
 - `test-dev-win` - `r-devel` on Windows
+- `test-old-win` - `r-oldrel` on Windows
 - `test-rel-osx` - MacOSX build not yet deployed, see [#3326](https://github.com/Rdatatable/data.table/issues/3326) for status
 
 Artifacts:
@@ -25,7 +26,7 @@ Artifacts:
 - [html vignettes](https://rdatatable.gitlab.io/data.table/library/data.table/doc/index.html)
 - R packages repository for `data.table` and all _Suggests_ dependencies, url: `https://Rdatatable.gitlab.io/data.table`
   - sources
-  - Windows binaries for `r-release` and `r-devel`
+  - Windows binaries for `r-release`, `r-devel` and `r-oldrel`
 - [CRAN-like homepage](https://rdatatable.gitlab.io/data.table/web/packages/data.table/index.html)
 - [CRAN-like checks results](https://rdatatable.gitlab.io/data.table/web/checks/check_results_data.table.html) - note that all artifacts, including check results page, are being published only when all test jobs successfully pass, thus one will not see an _ERROR_ status there (unless error happened on a job marked as `allow_failure`).
 - [docker images](https://gitlab.com/Rdatatable/data.table/container_registry) - copy/paste-able `docker pull` commands can be found at the bottom of our [CRAN-like homepage](https://rdatatable.gitlab.io/data.table/web/packages/data.table/index.html)

.ci/publish.R

@@ -1,12 +1,17 @@
 format.deps <- function(file, which) {
   deps.raw = read.dcf(file, fields=which)[[1L]]
   if (all(is.na(deps.raw))) return(character())
+  deps.raw = gsub("\n", " ", deps.raw, fixed=TRUE)
   deps.full = trimws(strsplit(deps.raw, ", ", fixed=TRUE)[[1L]])
   deps = trimws(sapply(strsplit(deps.full, "(", fixed=TRUE), `[[`, 1L))
+  deps.full = gsub(">=", "&ge;", deps.full, fixed=TRUE)
+  deps.full = gsub("<=", "&le;", deps.full, fixed=TRUE)
+  if (any(grepl(">", deps.full, fixed=TRUE), grepl("<", deps.full, fixed=TRUE), grepl("=", deps.full, fixed=TRUE)))
+    stop("formatting dependencies version for CRAN-line package website failed because some dependencies have version defined using operators other than >= and <=")
   names(deps.full) <- deps
   base.deps = c("R", unlist(tools:::.get_standard_package_names(), use.names = FALSE))
   ans = sapply(deps, function(x) {
-    if (x %in% base.deps) deps.full[[x]]
+    if (x %in% base.deps) deps.full[[x]] ## base R packages are not linked
     else sprintf("<a href=\"../%s/index.html\">%s</a>", x, deps.full[[x]])
   })
   sprintf("<tr><td>%s:</td><td>%s</td></tr>", which, paste(ans, collapse=", "))
@@ -26,6 +31,39 @@ format.bins <- function(ver, bin_ver, cran.home, os.type, pkg, version, repodir)
   paste(ans[fe], collapse=", ")
 }
 
+format.entry <- function(field, dcf, url=FALSE) {
+  if (field %in% colnames(dcf)) {
+    value = gsub("\n", " ", dcf[,field], fixed=TRUE)
+    if (url) {
+      urls = trimws(strsplit(value, ",", fixed=TRUE)[[1L]])
+      value = paste(sprintf("<a href=\"%s\">%s</a>", urls, urls), collapse=", ")
+    }
+    sprintf("<tr><td>%s:</td><td>%s</td></tr>", field, value)
+  }
+}
+format.maintainer <- function(dcf) {
+  if ("Maintainer" %in% colnames(dcf)) {
+    text2html = function(x) {
+      # https://stackoverflow.com/a/64446320/2490497
+      splitted <- strsplit(x, "")[[1L]]
+      intvalues <- as.hexmode(utf8ToInt(enc2utf8(x)))
+      paste(paste0("&#x", intvalues, ";"), collapse = "")
+    }
+    tmp = gsub("@", " at ", dcf[,"Maintainer"], fixed=TRUE)
+    sep = regexpr("<", tmp, fixed=TRUE)
+    name = trimws(substr(tmp, 1L, sep-1L))
+    mail = text2html(trimws(substr(tmp, sep, nchar(tmp))))
+    sprintf("<tr><td>Maintainer:</td><td>%s  %s</td></tr>", name, mail)
+  }
+}
+format.materials <- function() {
+  return(NULL) ## TODO
+  value = NA
+  #NEWS
+  #README
+  sprintf("<tr><td>Materials:</td><td>%s</td></tr>", value)
+}
+
 package.index <- function(package, lib.loc, repodir="bus/integration/cran") {
   file = system.file("DESCRIPTION", package=package, lib.loc=lib.loc)
   dcf = read.dcf(file)
@@ -40,21 +78,31 @@ package.index <- function(package, lib.loc, repodir="bus/integration/cran") {
     format.deps(file, "LinkingTo"),
     format.deps(file, "Suggests"),
     format.deps(file, "Enhances"),
+    if ("Built" %in% colnames(dcf)) sprintf("<tr><td>Built:</td><td>%s</td></tr>", substr(trimws(strsplit(dcf[,"Built"], ";", fixed=TRUE)[[1L]][[3L]]), 1L, 10L)),
+    if ("Author" %in% colnames(dcf)) sprintf("<tr><td>Author:</td><td>%s</td></tr>", dcf[,"Author"]),
+    format.maintainer(dcf),
+    format.entry("BugReports", dcf, url=TRUE),
+    format.entry("License", dcf),
+    format.entry("URL", dcf, url=TRUE),
+    format.entry("NeedsCompilation", dcf),
+    format.entry("SystemRequirements", dcf),
+    format.materials(), ## TODO
     if (pkg=="data.table") sprintf("<tr><td>Checks:</td><td><a href=\"../../checks/check_results_%s.html\">%s results</a></td></tr>", pkg, pkg)
   )
   vign = tools::getVignetteInfo(pkg, lib.loc=lib.loc)
-  r_bin_ver = Sys.getenv("R_BIN_VERSION")
-  r_devel_bin_ver = Sys.getenv("R_DEVEL_BIN_VERSION")
-  stopifnot(nzchar(r_bin_ver), nzchar(r_devel_bin_ver))
+  r_rel_ver = Sys.getenv("R_REL_VERSION")
+  r_devel_ver = Sys.getenv("R_DEVEL_VERSION")
+  r_oldrel_ver = Sys.getenv("R_OLDREL_VERSION")
+  stopifnot(nzchar(r_rel_ver), nzchar(r_devel_ver), nzchar(r_oldrel_ver))
   cran.home = "../../.."
   tbl.dl = c(
     sprintf("<tr><td> Reference manual: </td><td> <a href=\"%s.pdf\">%s.pdf</a>, <a href=\"%s/library/%s/html/00Index.html\">00Index.html</a> </td></tr>", pkg, pkg, cran.home, pkg),
     if (nrow(vign)) sprintf("<tr><td>Vignettes:</td><td>%s</td></tr>", paste(sprintf("<a href=\"%s/library/data.table/doc/%s\">%s</a><br/>", cran.home, vign[,"PDF"], vign[,"Title"]), collapse="\n")), # location unline cran web/pkg/vignettes to not duplicate content, documentation is in ../../../library
     sprintf("<tr><td> Package source: </td><td> <a href=\"%s/src/contrib/%s_%s.tar.gz\"> %s_%s.tar.gz </a> </td></tr>", cran.home,pkg, version, pkg, version),
-    sprintf("<tr><td> Windows binaries: </td><td> %s </td></tr>", format.bins(ver=c("r-devel","r-release"), bin_ver=c(r_devel_bin_ver,r_bin_ver), cran.home=cran.home, os.type="windows", pkg=pkg, version=version, repodir=repodir)),
-    sprintf("<tr><td> OS X binaries: </td><td> %s </td></tr>", format.bins(ver=c("r-devel","r-release"), bin_ver=c(r_devel_bin_ver, r_bin_ver), cran.home=cran.home, os.type="macosx", pkg=pkg, version=version, repodir=repodir))
+    sprintf("<tr><td> Windows binaries: </td><td> %s </td></tr>", format.bins(ver=c("r-devel","r-release","r-oldrel"), bin_ver=c(r_devel_ver, r_rel_ver, r_oldrel_ver), cran.home=cran.home, os.type="windows", pkg=pkg, version=version, repodir=repodir)),
+    sprintf("<tr><td> macOS binaries: </td><td> %s </td></tr>", format.bins(ver=c("r-release","r-oldrel"), bin_ver=c(r_rel_ver, r_oldrel_ver), cran.home=cran.home, os.type="macosx", pkg=pkg, version=version, repodir=repodir))
   )
-  if (pkg=="data.table") {
+  if (pkg=="data.table") { ## docker images
     registry = Sys.getenv("CI_REGISTRY", "registry.gitlab.com")
     namespace = Sys.getenv("CI_PROJECT_NAMESPACE", "Rdatatable")
     project = Sys.getenv("CI_PROJECT_NAME", "data.table")
@@ -74,7 +122,7 @@ package.index <- function(package, lib.loc, repodir="bus/integration/cran") {
     "<style type=\"text/css\">table td { vertical-align: top; }</style>",
     "</head>",
     "<body>",
-    sprintf("<h2>%s</h2>", dcf[,"Title"]),
+    sprintf("<h2>%s: %s</h2>", pkg, dcf[,"Title"]),
     sprintf("<p>%s</p>", dcf[,"Description"]),
     sprintf("<table summary=\"Package %s summary\">", pkg),
     tbl,
@@ -117,7 +165,48 @@ doc.copy <- function(repodir="bus/integration/cran"){
   c(ans1, ans2)
 }
 
-plat <- function(x) if (grepl("^.*win", x)) "Windows" else if (grepl("^.*osx", x)) "Mac OS X" else "Linux"
+plat <- function(x) if (grepl("^.*win", x)) "Windows" else if (grepl("^.*mac", x)) "macOS" else "Linux"
+
+r.ver <- function(x) {
+  tmp = strsplit(x, "-", fixed=TRUE)[[1L]]
+  if (length(tmp) < 2L) stop("test job names must be test-[r.version]-...")
+  v = tmp[2L]
+  if (identical(v, "rel")) "r-release"
+  else if (identical(v, "dev")) "r-devel"
+  else if (identical(v, "old")) "r-oldrel"
+  else {
+    if (grepl("\\D", v)) stop("second word in test job name must be rel/dev/old or numbers of R version")
+    paste0("r-", paste(strsplit(v, "")[[1L]], collapse="."))
+  }
+}
+
+# this for now is constant but when we move to independent pipelines (commit, daily, weekly) those values can be different
+pkg.version <- function(job, pkg) {
+  dcf = read.dcf(file.path("bus", job, paste(pkg, "Rcheck", sep="."), pkg, "DESCRIPTION"))
+  dcf[,"Version"]
+}
+pkg.revision <- function(job, pkg) {
+  dcf = read.dcf(file.path("bus", job, paste(pkg, "Rcheck", sep="."), pkg, "DESCRIPTION"))
+  if ("Revision" %in% colnames(dcf)) {
+    proj.url = Sys.getenv("CI_PROJECT_URL", "")
+    if (!nzchar(proj.url)) {
+      warning("pkg.revision was designed to be run on GLCI where CI_PROJECT_URL var is set, links to commits will not be produced for checks table")
+      substr(dcf[,"Revision"], 1, 7)
+    } else {
+      sprintf("<a href=\"%s\">%s</a>", file.path(proj.url, "-", "commit", dcf[,"Revision"]), substr(dcf[,"Revision"], 1, 7))
+    }
+  } else ""
+}
+pkg.flags <- function(job, pkg) {
+  cc = file.path("bus", job, paste(pkg, "Rcheck", sep="."), pkg, "cc") ## data.table style cc file
+  if (file.exists(cc)) {
+    d = readLines(cc)
+    w.cflags = substr(d, 1, 7)=="CFLAGS="
+    if (sum(w.cflags)==1L)
+      return(sub("CFLAGS=", "", d[w.cflags], fixed=TRUE))
+  }
+  ""
+}
 
 check.copy <- function(job, repodir="bus/integration/cran"){
   dir.create(job.checks<-file.path(repodir, "web", "checks", pkg<-"data.table", job), recursive=TRUE);
@@ -146,6 +235,39 @@ check.copy <- function(job, repodir="bus/integration/cran"){
   setNames(file.exists(file.path(job.checks, c(inst.check, routs))), c(inst.check, routs))
 }
 
+check.flavors <- function(jobs, repodir="bus/integration/cran") {
+  th = "<th>Flavor</th><th>R Version</th><th>OS Type</th><th>CPU Type</th><th>OS Info</th><th>CPU Info</th><th>Compilers</th>"
+  tbl = sprintf(
+    "<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>",
+    sub("test-", "", jobs, fixed=TRUE),
+    sapply(jobs, r.ver),
+    sapply(jobs, plat),
+    "", # "x86_64"
+    "", # "Debian GNU/Linux testing"
+    "", # "2x 8-core Intel(R) Xeon(R) CPU E5-2690 0 @ 2.90GHz"
+    "" # "GCC 10.2.0 (Debian 10.2.0-13)"
+  )
+  file = file.path(repodir, "web/checks", "check_flavors.html")
+  writeLines(c(
+    "<html>",
+    "<head>",
+    "<title>Package Check Flavors</title>",
+    "<link rel=\"stylesheet\" type=\"text/css\" href=\"../CRAN_web.css\"/>",
+    "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"/>",
+    "</head>",
+    "<body lang=\"en\">",
+    "<h2>Package Check Flavors</h2>",
+    sprintf("<p>Last updated on %s.</p>", format(Sys.time(), usetz=TRUE)),
+    "<table border=\"1\" summary=\"CRAN check flavors.\">",
+    "<tr>",th,"</tr>",
+    tbl,
+    "</table>",
+    "</body>",
+    "</html>"
+  ), file)
+  setNames(file.exists(file), file)
+}
+
 check.index <- function(pkg, jobs, repodir="bus/integration/cran") {
   status = function(x) if (grepl("^.*ERROR", x)) "ERROR" else if (grepl("^.*WARNING", x)) "WARNING" else if (grepl("^.*NOTE", x)) "NOTE" else if (grepl("^.*OK", x)) "OK" else NA_character_
   test.files = function(job, files, trim.name=FALSE, trim.exts=0L, pkg="data.table") {
@@ -186,30 +308,36 @@ check.index <- function(pkg, jobs, repodir="bus/integration/cran") {
     }
     memouts
   })
-  tbl = sprintf("<tr><td>%s</td><td>%s</td><td><a href=\"%s/%s/00install.out\">out</a></td><td><a href=\"%s/%s/00check.log\">%s</a></td><td>%s</td><td>%s</td></tr>",
-                sub("test-", "", jobs, fixed=TRUE),
-                sapply(jobs, plat),
-                pkg, jobs,
-                pkg, jobs, sapply(sapply(jobs, check.test, pkg="data.table"), status),
-                mapply(test.files, jobs, routs, trim.exts=2L), # 1st fail, 2nd Rout, keep just: tests_x64/main
-                mapply(test.files, jobs, memouts, trim.name=TRUE))
+  th = "<th>Flavor</th><th>Version</th><th>Revision</th><th>Install</th><th>Status</th><th>Flags</th><th>Rout.fail</th><th>Memtest</th>"
+  tbl = sprintf(
+    "<tr><td><a href=\"check_flavors.html\">%s</a></td><td>%s</td><td>%s</td><td><a href=\"%s/%s/00install.out\">out</a></td><td><a href=\"%s/%s/00check.log\">%s</a></td><td>%s</td><td>%s</td><td>%s</td></tr>",
+    sub("test-", "", jobs, fixed=TRUE),
+    sapply(jobs, pkg.version, pkg),
+    sapply(jobs, pkg.revision, pkg),
+    pkg, jobs, ## install
+    pkg, jobs, sapply(sapply(jobs, check.test, pkg="data.table"), status), ## check
+    sapply(jobs, pkg.flags, pkg),
+    mapply(test.files, jobs, routs, trim.exts=2L), # 1st fail, 2nd Rout, keep just: tests_x64/main
+    mapply(test.files, jobs, memouts, trim.name=TRUE)
+  )
   file = file.path(repodir, "web/checks", sprintf("check_results_%s.html", pkg))
-  writeLines(c("<html>",
-               "<head>",
-               sprintf("<title>Package Check Results for Package %s</title>", pkg),
-               "<link rel=\"stylesheet\" type=\"text/css\" href=\"../CRAN_web.css\"/>",
-               "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"/>",
-               "</head>",
-               "<body lang=\"en\">",
-               sprintf("<h2>Package Check Results for Package <a href=\"../packages/%s/index.html\"> %s </a> </h2>", pkg, pkg),
-               sprintf("<p>Last updated on %s.</p>", format(Sys.time(), usetz=TRUE)),
-               sprintf("<table border=\"1\" summary=\"CRAN check results for package %s\">", pkg),
-               "<tr><th>Test job</th><th>OS type</th><th>Install</th><th>Check</th><th>Rout.fail</th><th>Memtest</th></tr>",
-               tbl,
-               "</table>",
-               "</body>",
-               "</html>"),
-             file)
+  writeLines(c(
+    "<html>",
+    "<head>",
+    sprintf("<title>Package Check Results for Package %s</title>", pkg),
+    "<link rel=\"stylesheet\" type=\"text/css\" href=\"../CRAN_web.css\"/>",
+    "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"/>",
+    "</head>",
+    "<body lang=\"en\">",
+    sprintf("<h2>Package Check Results for Package <a href=\"../packages/%s/index.html\"> %s </a> </h2>", pkg, pkg),
+    sprintf("<p>Last updated on %s.</p>", format(Sys.time(), usetz=TRUE)),
+    sprintf("<table border=\"1\" summary=\"CRAN check results for package %s\">", pkg),
+    "<tr>",th,"</tr>",
+    tbl,
+    "</table>",
+    "</body>",
+    "</html>"
+  ), file)
   setNames(file.exists(file), file)
 }