Coerce 01

NEWS.md

@@ -192,24 +192,24 @@
     ```R
     set.seed(108)
     x = rnorm(1e6); n = 1e3
-    rollfun = function(x, n, FUN) {
-      ans = rep(NA_real_, nx<-length(x))
+    base_rollapply = function(x, n, FUN) {
+      nx = length(x)
+      ans = rep(NA_real_, nx)
       for (i in n:nx) ans[i] = FUN(x[(i-n+1):i])
       ans
     }
-    system.time(ans1<-rollfun(x, n, mean))
-    system.time(ans2<-zoo::rollapplyr(x, n, function(x) mean(x), fill=NA))
-    system.time(ans3<-zoo::rollmeanr(x, n, fill=NA))
-    system.time(ans4<-frollapply(x, n, mean))
-    system.time(ans5<-frollmean(x, n))
-    sapply(list(ans2,ans3,ans4,ans5), all.equal, ans1)
+    system.time(base_rollapply(x, n, mean))
+    system.time(zoo::rollapplyr(x, n, function(x) mean(x), fill=NA))
+    system.time(zoo::rollmeanr(x, n, fill=NA))
+    system.time(frollapply(x, n, mean))
+    system.time(frollmean(x, n))
 
     ### fun             mean     sum  median
-    # base rollfun     8.815   5.151  60.175
+    # base_rollapply   8.815   5.151  60.175
     # zoo::rollapply  34.373  27.837  88.552
-    # zoo::roll[fun]   0.215   0.185      NA
+    # zoo::roll[fun]   0.215   0.185      NA   ## median not fully supported
     # frollapply       5.404   1.419  56.475
-    # froll[fun]       0.003   0.002      NA
+    # froll[fun]       0.003   0.002      NA   ## median not yet supported
     ```
 
 28. `setnames()` now accepts functions in `old=` and `new=`, [#3703](https://github.com/Rdatatable/data.table/issues/3703). Thanks @smingerson for the feature request and @shrektan for the PR.
@@ -221,9 +221,11 @@
     # [1] "A" "B" "C"
     setnames(DT, 2:3, tolower)
     names(DT)
-    # [1] "A" "b" "c"
+    # [1] "a" "b" "c"
     ```
 
+29. `:=` and `set()` now use zero-copy type coercion. Accordingly, `DT[..., integerColumn:=0]` and `set(DT,i,j,0)` no longer warn about the `0` ('numeric') needing to be `0L` ('integer') because there is no longer any time or space used for this coercion. The old long warning was off-putting to new users ("what and why L?"), whereas advanced users appreciated the old warning so they could avoid the coercion. Although the time and space for one coercion in a single call is unmeasurably small, when placed in a loop the small overhead of any allocation on R's heap could start to become noticeable (more so for `set()` whose purpose is low-overhead looping). Further, when assigning a value across columns of varying types, it could be inconvenient to supply the correct type for every column. Hence, zero-copy coercion was introduced to satisfy all these requirements. A warning is still issued, as before, when fractional data is discarded; e.g. when 3.14 is assigned to an integer column. Zero-copy coercion applies to length>1 vectors as well as length-1 vectors.
+
 ## BUG FIXES
 
 1. `first`, `last`, `head` and `tail` by group no longer error in some cases, [#2030](https://github.com/Rdatatable/data.table/issues/2030) [#3462](https://github.com/Rdatatable/data.table/issues/3462). Thanks to @franknarf1 for reporting.
@@ -327,6 +329,8 @@
 
 44. Grouping could create a `malformed factor` and/or segfault when the factors returned by each group did not have identical levels, [#2199](https://github.com/Rdatatable/data.table/issues/2199) and [#2522](https://github.com/Rdatatable/data.table/issues/2522). Thanks to Václav Hausenblas, @franknarf1, @ben519, and @Henrik-P for reporting.
 
+45. `rbindlist` (and printing a `data.table` with over 100 rows because that uses `rbindlist(head, tail)`) could error with `malformed factor` for unordered factor columns containing a used `NA_character_` level, [#3915](https://github.com/Rdatatable/data.table/issues/3915). This is an unusual input for unordered factors because NA_integer_ is recommended by default in R. Thanks to @sindribaldur for reporting.
+
 ## NOTES
 
 1. `rbindlist`'s `use.names="check"` now emits its message for automatic column names (`"V[0-9]+"`) too, [#3484](https://github.com/Rdatatable/data.table/pull/3484). See news item 5 of v1.12.2 below.

R/bmerge.R

@@ -75,12 +75,12 @@ bmerge = function(i, x, icols, xcols, roll, rollends, nomatch, mult, ops, verbos
     if (xclass=="character" || iclass=="character" ||
         xclass=="logical" || iclass=="logical" ||
         xclass=="factor" || iclass=="factor") {
-      if (anyNA(i[[ic]]) && all(is.na(i[[ic]]))) { # TODO: allNA function in C
+      if (anyNA(i[[ic]]) && allNA(i[[ic]])) {
         if (verbose) cat("Coercing all-NA i.",names(i)[ic]," (",iclass,") to type ",xclass," to match type of x.",names(x)[xc],".\n",sep="")
         set(i, j=ic, value=match.fun(paste0("as.", xclass))(i[[ic]]))
         next
       }
-      else if (anyNA(x[[xc]]) && all(is.na(x[[xc]]))) {
+      else if (anyNA(x[[xc]]) && allNA(x[[xc]])) {
         if (verbose) cat("Coercing all-NA x.",names(x)[xc]," (",xclass,") to type ",iclass," to match type of i.",names(i)[ic],".\n",sep="")
         set(x, j=xc, value=match.fun(paste0("as.", iclass))(x[[xc]]))
         next

R/fread.R

@@ -118,7 +118,7 @@ yaml=FALSE, autostart=NA, tmpdir=tempdir())
   }
   if (!missing(autostart)) warning("'autostart' is now deprecated and ignored. Consider skip='string' or skip=n");
   if (is.logical(colClasses)) {
-    if (!all(is.na(colClasses))) stop("colClasses is type 'logical' which is ok if all NA but it has some TRUE or FALSE values in it which is not allowed. Please consider the drop= or select= argument instead. See ?fread.")
+    if (!allNA(colClasses)) stop("colClasses is type 'logical' which is ok if all NA but it has some TRUE or FALSE values in it which is not allowed. Please consider the drop= or select= argument instead. See ?fread.")
     colClasses = NULL
   }
   if (!is.null(colClasses) && is.atomic(colClasses)) {

R/test.data.table.R

@@ -157,7 +157,7 @@ test.data.table = function(verbose=FALSE, pkg="pkg", silent=FALSE, with.other.pa
   #  inittime=PS_rss=GC_used=GC_max_used=NULL
   #  m = fread("memtest.csv")[inittime==.inittime]
   #  if (nrow(m)) {
-  #    ps_na = all(is.na(m[["PS_rss"]])) # OS with no 'ps -o rss R' support
+  #    ps_na = allNA(m[["PS_rss"]]) # OS with no 'ps -o rss R' support
   #    grDevices::png("memtest.png")
   #    p = graphics::par(mfrow=c(if (ps_na) 2 else 3, 2))
   #    if (!ps_na) {
@@ -282,9 +282,9 @@ test = function(num,x,y=TRUE,error=NULL,warning=NULL,output=NULL,notOutput=NULL,
   if (!missing(error) && !missing(y))
     stop("Test ",numStr," is invalid: when error= is provided it does not make sense to pass y as well")  # nocov
 
-  string_match = function(x, y) {
-    length(grep(x,y,fixed=TRUE)) ||                    # try treating x as literal first; useful for most messages containing ()[]+ characters
-    length(tryCatch(grep(x,y), error=function(e)NULL)) # otherwise try x as regexp
+  string_match = function(x, y, ignore.case=FALSE) {
+    length(grep(x, y, fixed=TRUE)) ||  # try treating x as literal first; useful for most messages containing ()[]+ characters
+    length(tryCatch(grep(x, y, ignore.case=ignore.case), error=function(e)NULL))  # otherwise try x as regexp
   }
 
   xsub = substitute(x)
@@ -364,10 +364,10 @@ test = function(num,x,y=TRUE,error=NULL,warning=NULL,output=NULL,notOutput=NULL,
       fail = TRUE
       # nocov end
     }
-    if (length(notOutput) && string_match(notOutput, out)) {
+    if (length(notOutput) && string_match(notOutput, out, ignore.case=TRUE)) {
       # nocov start
       cat("Test",numStr,"produced output but should not have:\n")
-      cat("Expected absent: <<",gsub("\n","\\\\n",notOutput),">>\n",sep="")
+      cat("Expected absent (case insensitive): <<",gsub("\n","\\\\n",notOutput),">>\n",sep="")
       cat("Observed: <<",gsub("\n","\\\\n",out),">>\n",sep="")
       fail = TRUE
       # nocov end

R/utils.R

@@ -12,6 +12,7 @@ if (base::getRversion() < "3.5.0") {
 }
 isTRUEorNA    = function(x) is.logical(x) && length(x)==1L && (is.na(x) || x)
 isTRUEorFALSE = function(x) is.logical(x) && length(x)==1L && !is.na(x)
+allNA = function(x) .Call(C_allNAR, x)
 
 if (base::getRversion() < "3.2.0") {  # Apr 2015
   isNamespaceLoaded = function(x) x %chin% loadedNamespaces()