Return value of `cpp_dfm` can be invalid, non-deterministic

[jaganmn]

Describe the bug

cpp_dfm returns an invalid dgCMatrix inside of dfm.tokens_xptr. In a debugger, evaluating the call to cpp_dfm a second time, interactively, gives a different (this time valid) object. So the return value can be not only invalid but also non-deterministic.

Reproducible code

(Original, non-minimal example is here: https://stackoverflow.com/q/78841310/12685768)

library(quanteda)
d <- dget("dput-frame.txt")
x <- corpus(d, docid_field = "filename", text_field = "content")
y <- dfm(tokens(x))

dput-frame.txt

> library(quanteda)
Package version: 4.0.2
Unicode version: 15.0
ICU version: 73.2
Parallel computing: disabled
See https://quanteda.io for tutorials and examples.
> d <- dget("dput-frame.txt")
> x <- corpus(d, docid_field = "filename", text_field = "content")
> y <- dfm(tokens(x))
Error in validObject(.Object) : 
  invalid class "dfm" object: first element of 'p' slot is not 0
> traceback()
12: stop(msg, ": ", errors, domain = NA)
11: validObject(.Object)
10: .nextMethod(.Object = .Object, ... = ...)
9: callNextMethod()
8: initialize(value, ...)
7: initialize(value, ...)
6: new("dfm", as(as(as(x, "CsparseMatrix"), "generalMatrix"), "dMatrix"), 
       docvars = docvars, meta = make_meta("dfm", inherit = meta, 
           ...))
5: build_dfm(temp, colnames(temp), docvars = get_docvars(x, user = TRUE, 
       system = TRUE), meta = attrs[["meta"]])
4: dfm.tokens_xptr(as.tokens_xptr(x), tolower = tolower, remove_padding = remove_padding, 
       verbose = verbose, ...)
3: dfm(as.tokens_xptr(x), tolower = tolower, remove_padding = remove_padding, 
       verbose = verbose, ...)
2: dfm.tokens(tokens(x))
1: dfm(tokens(x))
> debugonce(quanteda:::dfm.tokens_xptr)
> y <- dfm(tokens(x))
debugging in: dfm.tokens_xptr(as.tokens_xptr(x), tolower = tolower, remove_padding = remove_padding, 
    verbose = verbose, ...)
debug: {
    if (is.null(global$object_class)) {
        global$object_class <- class(x)[1]
        global$proc_time <- proc.time()
    }
    check_dots(...)
    if (verbose) 
        catm("Creating a dfm from a", global$object_class, "object...\n")
    x <- as.tokens_xptr(x)
    if (tolower) 
        x <- tokens_tolower(x)
    if (remove_padding) 
        x <- tokens_remove(x, "", valuetype = "fixed")
    attrs <- attributes(x)
    temp <- t(cpp_dfm(x, attrs$meta$object$what == "dictionary"))
    result <- build_dfm(temp, colnames(temp), docvars = get_docvars(x, 
        user = TRUE, system = TRUE), meta = attrs[["meta"]])
    if (verbose) {
        catm(" ...complete, elapsed time:", format((proc.time() - 
            global$proc_time)[3], digits = 3), "seconds.\n")
        catm("Finished constructing a", paste(format(dim(result), 
            big.mark = ",", trim = TRUE), collapse = " x "), 
            "sparse dfm.\n")
    }
    global$object_class <- NULL
    return(result)
}
Browse[1]> n
debug: if (is.null(global$object_class)) {
    global$object_class <- class(x)[1]
    global$proc_time <- proc.time()
}
Browse[1]> n
debug: check_dots(...)
Browse[1]> n
debug: if (verbose) catm("Creating a dfm from a", global$object_class, 
    "object...\n")
Browse[1]> n
debug: x <- as.tokens_xptr(x)
Browse[1]> n
debug: if (tolower) x <- tokens_tolower(x)
Browse[1]> n
debug: x <- tokens_tolower(x)
Browse[1]> n
debug: if (remove_padding) x <- tokens_remove(x, "", valuetype = "fixed")
Browse[1]> n
debug: attrs <- attributes(x)
Browse[1]> n
debug: temp <- t(cpp_dfm(x, attrs$meta$object$what == "dictionary"))
Browse[1]> validObject(cpp_dfm(x, attrs$meta$object$what == "dictionary"))
Error in validObject(cpp_dfm(x, attrs$meta$object$what == "dictionary")) : 
  invalid class "dgCMatrix" object: first differences of 'p' slot exceed Dim[1]
Browse[1]> validObject(cpp_dfm(x, attrs$meta$object$what == "dictionary"))
[1] TRUE
Browse[1]> validObject(cpp_dfm(x, attrs$meta$object$what == "dictionary"))
[1] TRUE
Browse[1]>

## System information

> sessionInfo()
R Under development (unstable) (2024-08-07 r86984)
Platform: aarch64-apple-darwin22.6.0
Running under: macOS Ventura 13.6.7

Matrix products: default
BLAS:   /usr/local/lib/R/lib/libRblas.dylib 
LAPACK: /usr/local/lib/R/lib/libRlapack.dylib;  LAPACK version 3.12.0

locale:
[1] C/en_CA.UTF-8/en_CA.UTF-8/C/en_CA.UTF-8/en_CA.UTF-8

time zone: America/Toronto
tzcode source: internal

attached base packages:
[1] stats     graphics  grDevices utils     datasets  methods   base     

other attached packages:
[1] quanteda_4.0.2

loaded via a namespace (and not attached):
 [1] compiler_4.5.0  magrittr_2.0.3  Matrix_1.8-0    cli_3.6.3      
 [5] tools_4.5.0     fastmatch_1.1-4 Rcpp_1.0.13     stringi_1.8.4  
 [9] grid_4.5.0      stopwords_2.3   lifecycle_1.0.4 rlang_1.1.4    
[13] lattice_0.22-6

[koheiw]

It seems that \\ufeff between line 95 and 96 is causing the problem. It appears like a padding in the last token of toks but it is a different character.

require(quanteda)
#> Loading required package: quanteda
#> Package version: 4.0.2
#> Unicode version: 15.1
#> ICU version: 74.1
#> Parallel computing: 16 of 16 threads used.
#> See https://quanteda.io for tutorials and examples.

require(quanteda.textstats)
#> Loading required package: quanteda.textstats

file <- "C:/Users/watan/Downloads/1576940_10-K_2019-01-01_0001047469-12-002544.txt"
line <- readLines(file)

txt <- paste(line[95:96], collapse = "\n")
stringi::stri_escape_unicode(txt)
#> [1] "We are subject to numerous local, state, federal and other statutes, ordinances, rules and regulations concerning zoning, development, building design, construction and similar matters which impose restrictive zoning and density requirements in order to limit the number of homes that can eventually be built within the boundaries of a particular area. Projects that are not entitled may be subjected to periodic delays, changes in use, less intensive development or elimination of development in certain specific areas due to government regulations. We may also be subject to periodic delays or may be precluded entirely from developing in certain communities due to building moratoriums or \\u201cslow-growth\\u201d or \\u201cno-growth\\u201d initiatives that could be implemented in the future. Local and state governments also have broad discretion regarding the imposition of development fees for projects in their jurisdiction. Projects for which we have received land use and development entitlements or approvals may still require a variety of other governmental approvals and permits during the development process and can also be impacted adversely by unforeseen municipal, regulatory, health, safety and welfare issues, which can further delay these projects or prevent their development.\\n\\ufeff"

corp <- corpus(txt)
toks <- tokens(corp)
# dfm(toks) # error
print(toks, -1, -1)
#> Tokens consisting of 1 document.
#> text1 :
#>   [1] "We"           "are"          "subject"      "to"           "numerous"    
#>   [6] "local"        ","            "state"        ","            "federal"     
#>  [11] "and"          "other"        "statutes"     ","            "ordinances"  
#>  [16] ","            "rules"        "and"          "regulations"  "concerning"  
#>  [21] "zoning"       ","            "development"  ","            "building"    
#>  [26] "design"       ","            "construction" "and"          "similar"     
#>  [31] "matters"      "which"        "impose"       "restrictive"  "zoning"      
#>  [36] "and"          "density"      "requirements" "in"           "order"       
#>  [41] "to"           "limit"        "the"          "number"       "of"          
#>  [46] "homes"        "that"         "can"          "eventually"   "be"          
#>  [51] "built"        "within"       "the"          "boundaries"   "of"          
#>  [56] "a"            "particular"   "area"         "."            "Projects"    
#>  [61] "that"         "are"          "not"          "entitled"     "may"         
#>  [66] "be"           "subjected"    "to"           "periodic"     "delays"      
#>  [71] ","            "changes"      "in"           "use"          ","           
#>  [76] "less"         "intensive"    "development"  "or"           "elimination" 
#>  [81] "of"           "development"  "in"           "certain"      "specific"    
#>  [86] "areas"        "due"          "to"           "government"   "regulations" 
#>  [91] "."            "We"           "may"          "also"         "be"          
#>  [96] "subject"      "to"           "periodic"     "delays"       "or"          
#> [101] "may"          "be"           "precluded"    "entirely"     "from"        
#> [106] "developing"   "in"           "certain"      "communities"  "due"         
#> [111] "to"           "building"     "moratoriums"  "or"           "“"           
#> [116] "slow-growth"  "”"            "or"           "“"            "no-growth"   
#> [121] "”"            "initiatives"  "that"         "could"        "be"          
#> [126] "implemented"  "in"           "the"          "future"       "."           
#> [131] "Local"        "and"          "state"        "governments"  "also"        
#> [136] "have"         "broad"        "discretion"   "regarding"    "the"         
#> [141] "imposition"   "of"           "development"  "fees"         "for"         
#> [146] "projects"     "in"           "their"        "jurisdiction" "."           
#> [151] "Projects"     "for"          "which"        "we"           "have"        
#> [156] "received"     "land"         "use"          "and"          "development" 
#> [161] "entitlements" "or"           "approvals"    "may"          "still"       
#> [166] "require"      "a"            "variety"      "of"           "other"       
#> [171] "governmental" "approvals"    "and"          "permits"      "during"      
#> [176] "the"          "development"  "process"      "and"          "can"         
#> [181] "also"         "be"           "impacted"     "adversely"    "by"          
#> [186] "unforeseen"   "municipal"    ","            "regulatory"   ","           
#> [191] "health"       ","            "safety"       "and"          "welfare"     
#> [196] "issues"       ","            "which"        "can"          "further"     
#> [201] "delay"        "these"        "projects"     "or"           "prevent"     
#> [206] "their"        "development"  "."            ""

Workaround is applying stringi::stri_replace_all_regex(txt, "\ufeff", "").

txt2 <- stringi::stri_replace_all_regex(txt, "\ufeff", "")
corp2 <- corpus(txt2)
toks2 <- tokens(corp2)
print(toks2, -1, -1)
#> Tokens consisting of 1 document.
#> text1 :
#>   [1] "We"           "are"          "subject"      "to"           "numerous"    
#>   [6] "local"        ","            "state"        ","            "federal"     
#>  [11] "and"          "other"        "statutes"     ","            "ordinances"  
#>  [16] ","            "rules"        "and"          "regulations"  "concerning"  
#>  [21] "zoning"       ","            "development"  ","            "building"    
#>  [26] "design"       ","            "construction" "and"          "similar"     
#>  [31] "matters"      "which"        "impose"       "restrictive"  "zoning"      
#>  [36] "and"          "density"      "requirements" "in"           "order"       
#>  [41] "to"           "limit"        "the"          "number"       "of"          
#>  [46] "homes"        "that"         "can"          "eventually"   "be"          
#>  [51] "built"        "within"       "the"          "boundaries"   "of"          
#>  [56] "a"            "particular"   "area"         "."            "Projects"    
#>  [61] "that"         "are"          "not"          "entitled"     "may"         
#>  [66] "be"           "subjected"    "to"           "periodic"     "delays"      
#>  [71] ","            "changes"      "in"           "use"          ","           
#>  [76] "less"         "intensive"    "development"  "or"           "elimination" 
#>  [81] "of"           "development"  "in"           "certain"      "specific"    
#>  [86] "areas"        "due"          "to"           "government"   "regulations" 
#>  [91] "."            "We"           "may"          "also"         "be"          
#>  [96] "subject"      "to"           "periodic"     "delays"       "or"          
#> [101] "may"          "be"           "precluded"    "entirely"     "from"        
#> [106] "developing"   "in"           "certain"      "communities"  "due"         
#> [111] "to"           "building"     "moratoriums"  "or"           "“"           
#> [116] "slow-growth"  "”"            "or"           "“"            "no-growth"   
#> [121] "”"            "initiatives"  "that"         "could"        "be"          
#> [126] "implemented"  "in"           "the"          "future"       "."           
#> [131] "Local"        "and"          "state"        "governments"  "also"        
#> [136] "have"         "broad"        "discretion"   "regarding"    "the"         
#> [141] "imposition"   "of"           "development"  "fees"         "for"         
#> [146] "projects"     "in"           "their"        "jurisdiction" "."           
#> [151] "Projects"     "for"          "which"        "we"           "have"        
#> [156] "received"     "land"         "use"          "and"          "development" 
#> [161] "entitlements" "or"           "approvals"    "may"          "still"       
#> [166] "require"      "a"            "variety"      "of"           "other"       
#> [171] "governmental" "approvals"    "and"          "permits"      "during"      
#> [176] "the"          "development"  "process"      "and"          "can"         
#> [181] "also"         "be"           "impacted"     "adversely"    "by"          
#> [186] "unforeseen"   "municipal"    ","            "regulatory"   ","           
#> [191] "health"       ","            "safety"       "and"          "welfare"     
#> [196] "issues"       ","            "which"        "can"          "further"     
#> [201] "delay"        "these"        "projects"     "or"           "prevent"     
#> [206] "their"        "development"  "."

txt2 <- paste(line, collapse = "\n")
txt2 <- stringi::stri_replace_all_regex(txt2, "\ufeff", "")
corp2 <- corpus(txt2)
toks2 <- tokens(corp2)
dfm(toks2)
#> Document-feature matrix of: 1 document, 2,759 features (0.00% sparse) and 0 docvars.
#>        features
#> docs    cik  : company name form type filing date accession number
#>   text1   1 20      20    1   14    3      2    8         1     29
#> [ reached max_nfeat ... 2,749 more features ]

[koheiw]

\ufeff (non-breaking space) is causing problems because it is removed by stringi functions when it appears as a single code point.

require(stringi)
#> Loading required package: stringi

toks <- tokens("A a b \n\ufeff", 
               padding = TRUE, xptr = FALSE)
#dfm(toks, tolower = TRUE) # error
dfm(toks, tolower = FALSE) # works
#> Document-feature matrix of: 1 document, 4 features (0.00% sparse) and 0 docvars.
#>        features
#> docs    A a b  
#>   text1 1 1 1 1

types(tokens_tolower(toks)) == "\ufeff"
#> [1] FALSE FALSE

types(toks) == "\ufeff"
#> [1] FALSE FALSE FALSE  TRUE

stri_escape_unicode(stri_trans_tolower("\n\ufeff"))
#> [1] "\\n\\ufeff"

stri_escape_unicode(stri_trans_tolower("\ufeff"))
#> [1] ""

stri_escape_unicode(stri_trans_nfc("\n\ufeff"))
#> [1] "\\n\\ufeff"

stri_escape_unicode(stri_trans_nfc("\ufeff"))
#> [1] ""

^{Created on 2024-08-11 with reprex v2.1.1}