Revdep failure: data.table

[MichaelChirico]

Investigating this series of errors:

Test 1102.14 ran without errors but failed check that x equals y:
> x = dcast(DT, a ~ b, drop = FALSE, fun.aggregate = length, value.var = "b") 
   a v w x y z
1: a 0 0 2 0 0
2: b 1 0 0 2 0
3: c 1 0 2 1 1
4: d 0 0 0 0 0
5: e 0 0 0 0 0
    [Key=a Types=int,int,int,int,int,int Classes=fac,int,int,int,int,int]
1:                                                                       
2:                                                                       
3:                                                                       
4:                                                                       
5:                                                                       
> y = data.table(a = factor(letters[1:5], levels = letters[1:5]), v = INT(0,      1, 0, 0, 0), w = INT(1, 1, 1, 0, 0), x = INT(0, 0, 1, 0,      0), y = INT(2, 1, 1, 0, 0), z = INT(0, 1, 0, 0, 0), key = "a") 
   a v w x y z
1: a 0 1 0 2 0
2: b 1 1 0 1 1
3: c 0 1 1 1 0
4: d 0 0 0 0 0
5: e 0 0 0 0 0
    [Key=a Types=int,int,int,int,int,int Classes=fac,int,int,int,int,int]
1:                                                                       
2:                                                                       
3:                                                                       
4:                                                                       
5:                                                                       
Column 'v': Mean relative difference: 1
Test 1102.15 ran without errors but failed check that x equals y:
> x = dcast(DT, a ~ b, drop = FALSE, value.var = "b", fun.aggregate = length) 
   a v w x y z
1: e 0 0 0 0 0
2: d 0 0 0 0 0
3: c 1 0 2 1 1
4: b 1 0 0 2 0
5: a 0 0 2 0 0
    [Key=a Types=int,int,int,int,int,int Classes=fac,int,int,int,int,int]
1:                                                                       
2:                                                                       
3:                                                                       
4:                                                                       
5:                                                                       
> y = data.table(a = factor(c("e", "d", "c", "b", "a"), levels = levels(DT$a)),      v = INT(0, 0, 0, 1, 0), w = INT(0, 0, 1, 1, 1), x = INT(0,          0, 1, 0, 0), y = INT(0, 0, 1, 1, 2), z = INT(0, 0, 0,          1, 0), key = "a") 
   a v w x y z
1: e 0 0 0 0 0
2: d 0 0 0 0 0
3: c 0 1 1 1 0
4: b 1 1 0 1 1
5: a 0 1 0 2 0
    [Key=a Types=int,int,int,int,int,int Classes=fac,int,int,int,int,int]
1:                                                                       
2:                                                                       
3:                                                                       
4:                                                                       
5:                                                                       
Column 'v': Mean relative difference: 1
Test 1102.16 ran without errors but failed check that x equals y:
> x = dcast(DT, a1 + a2 + a3 ~ b, drop = FALSE, value.var = "b")[c(1,      21, .N)] 
   a1 a2 a3    v    w    x    y    z
1:  a  f  a <NA> <NA> <NA> <NA> <NA>
2:  b  g  c <NA> <NA> <NA> <NA> <NA>
3:  e  j  c <NA> <NA> <NA> <NA> <NA>
    [Key=a1,a2,a3 Types=int,int,cha,int,int,int,int,int Classes=fac,fac,cha,fac,fac,fac,fac,fac]
1:                                                                                              
2:                                                                                              
3:                                                                                              
> y = data.table(a1 = factor(c("a", "b", "e"), levels = letters[1:5]),      a2 = factor(c("f", "g", "j"), levels = letters[6:10]), a3 = c("a",          "c", "c"), v = factor(NA, levels = letters[22:26]), w = factor(NA,          levels = letters[22:26]), x = factor(NA, levels = letters[22:26]),      y = factor(c(NA, "y", NA), levels = letters[22:26]), z = factor(NA,          levels = letters[22:26]), key = c("a1", "a2", "a3")) 
   a1 a2 a3    v    w    x    y    z
1:  a  f  a <NA> <NA> <NA> <NA> <NA>
2:  b  g  c <NA> <NA> <NA>    y <NA>
3:  e  j  c <NA> <NA> <NA> <NA> <NA>
    [Key=a1,a2,a3 Types=int,int,cha,int,int,int,int,int Classes=fac,fac,cha,fac,fac,fac,fac,fac]
1:                                                                                              
2:                                                                                              
3:                                                                                              
Column 'y': Levels not identical even after refactoring since trim.levels is TRUE
Test 1125 ran without errors but failed check that x equals y:
> x = rbindlist(list(DT1, DT2))[c(1, 2, .N - 1, .N), as.character(V1)] 
First 4 of 4 (type 'character'): 
[1] "id85645" "id80957" "id91193" "id98180"
> y = c("id85645", "id80957", "id73436", "id33445") 
First 4 of 4 (type 'character'): 
[1] "id85645" "id80957" "id73436" "id33445"
2 string mismatches
Error in eval(el, eval_frame) : '...' used in an incorrect context

Thu Mar  5 06:41:56 2026  endian==little, sizeof(long double)==16, capabilities('long.double')==TRUE, longdouble.digits==64, sizeof(pointer)==8, TZ==unset, Sys.timezone()=='Etc/UTC', Sys.getlocale()=='LC_CTYPE=C.UTF-8;LC_NUMERIC=C;LC_TIME=C.UTF-8;LC_COLLATE=C.UTF-8;LC_MONETARY=C.UTF-8;LC_MESSAGES=C.UTF-8;LC_PAPER=C.UTF-8;LC_NAME=C;LC_ADDRESS=C;LC_TELEPHONE=C;LC_MEASUREMENT=C.UTF-8;LC_IDENTIFICATION=C', l10n_info()=='MBCS=TRUE; UTF-8=TRUE; Latin-1=FALSE; codeset=UTF-8', getDTthreads()=='OpenMP version (_OPENMP)==201511; omp_get_num_procs()==2; R_DATATABLE_NUM_PROCS_PERCENT==unset (default 50); R_DATATABLE_NUM_THREADS==unset; R_DATATABLE_THROTTLE==unset (default 1024); omp_get_thread_limit()==2147483647; omp_get_max_threads()==2; OMP_THREAD_LIMIT==2; OMP_NUM_THREADS==unset; RestoreAfterFork==true; data.table is using 1 threads with throttle==1024. See ?setDTthreads.', .libPaths()=='/usr/local/lib/R/library', zlibVersion()==1.3.1 ZLIB_VERSION==1.3.1
Error in test.data.table() : 
  Failed in 18.9s elapsed (17.5s cpu) after test 1641.013 before the next test() call in /usr/local/lib/R/library/data.table/tests/tests.Rraw.bz2
Calls: test.data.table -> stopf -> raise_condition -> signal

[MichaelChirico]

It looks like the bit64::factor mask gives different output to base::factor:

set.seed(1L)
bit64::factor(sample(letters[1:3], 10L, replace=TRUE), levels=letters[1:5])
#  [1] c a a a b b b b c a
# Levels: a b c d e

set.seed(1L)
base::factor(sample(letters[1:3], 10L, replace=TRUE), levels=letters[1:5])
#  [1] a c a b a c c b b c
# Levels: a b c d e

[hcirellu]

Very interesting.

set.seed(1L)
sample(letters[1:3], 10L, replace=TRUE)
#  [1] "a" "c" "a" "b" "a" "c" "c" "b" "b" "c"
sample(letters[1:3], 10L, replace=TRUE)
#  [1] "c" "a" "a" "a" "b" "b" "b" "b" "c" "a"

That means that in bit64::factor the input is sampled twice.

[MichaelChirico]

Great observation. Yes, it's this:

trace(sample, at=1, quote(cat("called\n")))
foo = function(x) {
  x
  sys_call = sys.call()
  sys_call[[1]] = identity
  eval(sys_call, parent.frame())
}
foo(sample(10))
# Tracing sample(10) step 1 
# called
# Tracing sample(10) step 1 
# called
#  [1]  4  3  5  2  8  1  7  6  9 10

[MichaelChirico]

I think the same affects other functions like table():

set.seed(1); bit64::table(sample(as.integer64(1:10), replace=TRUE))
#  4  6  7  8  9 10 
#  2  2  1  1  2  2 

set.seed(1); base::table(sample(as.integer64(1:10), replace=TRUE))
# 1 2 3 4 5 7 9 
# 2 2 1 1 1 2 1 

[MichaelChirico]

#277 has a simple fix for factor(), I'm not sure what to do about a vararg case like table(), is the best thing really to do something like for (jj in seq_along(dots)) sys_call[[jj+1L]] = dots[[jj] ?

[hcirellu]

The problem that I identified is, that inherits already evaluates.

trace(sample, at=1, quote(cat("called sample\n")))
inherits(sample(1:10), "integer")
# Tracing sample(1:10) step 1 
# called sample
# [1] TRUE

A fix for factor could be:

factor = function(x=character(), levels, labels=levels, exclude=NA, ordered=is.ordered(x), nmax=NA) {
  force(x)
  if (!is.integer64(x)) {
    sys_call = match.call()
    sys_call[[1L]] = base::factor
    sys_call$x = x
...
}

For table this applies as well.

I think I've got a fix and I try to integrate it into #277.

[MichaelChirico]

The factor()/table() thing fixed almost everything, still one failure left in {data.table}:

Test 2181 produced 1 errors but expected 0
Expected: 
Observed: nothing to tabulate
V1
1
V1
0,1
0,2
a,b
a

Fri Mar  6 17:10:11 2026  endian==little, sizeof(long double)==16, capabilities('long.double')==TRUE, longdouble.digits==64, sizeof(pointer)==8, TZ==unset, Sys.timezone()=='Etc/UTC', Sys.getlocale()=='LC_CTYPE=C.UTF-8;LC_NUMERIC=C;LC_TIME=C.UTF-8;LC_COLLATE=C.UTF-8;LC_MONETARY=C.UTF-8;LC_MESSAGES=C.UTF-8;LC_PAPER=C.UTF-8;LC_NAME=C;LC_ADDRESS=C;LC_TELEPHONE=C;LC_MEASUREMENT=C.UTF-8;LC_IDENTIFICATION=C', l10n_info()=='MBCS=TRUE; UTF-8=TRUE; Latin-1=FALSE; codeset=UTF-8', getDTthreads()=='OpenMP version (_OPENMP)==201511; omp_get_num_procs()==2; R_DATATABLE_NUM_PROCS_PERCENT==unset (default 50); R_DATATABLE_NUM_THREADS==unset; R_DATATABLE_THROTTLE==unset (default 1024); omp_get_thread_limit()==2147483647; omp_get_max_threads()==2; OMP_THREAD_LIMIT==2; OMP_NUM_THREADS==unset; RestoreAfterFork==true; data.table is using 1 threads with throttle==1024. See ?setDTthreads.', .libPaths()=='/usr/local/lib/R/library', zlibVersion()==1.3.1 ZLIB_VERSION==1.3.1
Error in test.data.table() : 
  1 error out of 12106. Search tests/tests.Rraw.bz2 for test number 2181. Duration: 38.6s elapsed (36.8s cpu).
Calls: test.data.table -> stopf -> raise_condition -> signal
Execution halted

[MichaelChirico]

There's still an edge case to handle in our overwrite of table():

bit64::table(NULL)
# Error in table(NULL) : nothing to tabulate
base::table(NULL)
# < table of extent 0 >

[MichaelChirico]

Seems it was premature to close, whack-a-mole issue:

V1
1
V1
0,1
0,2
a,b
a

Fri Mar  6 17:37:29 2026  endian==little, sizeof(long double)==16, capabilities('long.double')==TRUE, longdouble.digits==64, sizeof(pointer)==8, TZ==unset, Sys.timezone()=='Etc/UTC', Sys.getlocale()=='LC_CTYPE=C.UTF-8;LC_NUMERIC=C;LC_TIME=C.UTF-8;LC_COLLATE=C.UTF-8;LC_MONETARY=C.UTF-8;LC_MESSAGES=C.UTF-8;LC_PAPER=C.UTF-8;LC_NAME=C;LC_ADDRESS=C;LC_TELEPHONE=C;LC_MEASUREMENT=C.UTF-8;LC_IDENTIFICATION=C', l10n_info()=='MBCS=TRUE; UTF-8=TRUE; Latin-1=FALSE; codeset=UTF-8', getDTthreads()=='OpenMP version (_OPENMP)==201511; omp_get_num_procs()==2; R_DATATABLE_NUM_PROCS_PERCENT==unset (default 50); R_DATATABLE_NUM_THREADS==unset; R_DATATABLE_THROTTLE==unset (default 1024); omp_get_thread_limit()==2147483647; omp_get_max_threads()==2; OMP_THREAD_LIMIT==2; OMP_NUM_THREADS==unset; RestoreAfterFork==true; data.table is using 1 threads with throttle==1024. See ?setDTthreads.', .libPaths()=='/usr/local/lib/R/library', zlibVersion()==1.3.1 ZLIB_VERSION==1.3.1
Caught 1 warning outside the test() calls:
   after test                                       warning
        <num>                                        <char>
1:     1730.5 NAs introduced by coercion to integer64 range
                                                                                                                                                                                                                                                                               calls
                                                                                                                                                                                                                                                                              <char>
1: test.data.table -> try -> tryCatch -> tryCatchList -> tryCatchOne -> doTryCatch -> withCallingHandlers -> sys.source -> eval -> eval -> data.table -> as.integer64 -> as.integer64.character -> .signalSimpleWarning -> withRestarts -> withOneRestart -> doWithOneRestart -> ...
Error in test.data.table() : 
  Tests succeeded, but non-test code caused warnings. Search tests/tests.Rraw.bz2 for tests shown above.
Calls: test.data.table -> stopf -> raise_condition -> signal