Refine memory budget use

R/TileDBArray.R

@@ -519,8 +519,9 @@ setMethod("[", "tiledb_array",
 
   ## A preference can be set in a local per-user configuration file; if no value
   ## is set a fallback from the TileDB config object is used. Note that this memory
-  ## budget (currently, at least) applies only to character columns.
-  memory_budget <- get_allocation_size_preference()
+  ## budget (currently, at least) applies only to character columns. We scale the total
+  ## budget by the number of variable sized column (where 'varnum' is NA)
+  memory_budget <- get_allocation_size_preference() / max(1, sum(is.na(allvarnum)), na.rm=TRUE)
 
   if (length(enckey) > 0) {
     if (length(tstamp) > 0) {
@@ -688,6 +689,7 @@ setMethod("[", "tiledb_array",
       getBuffer <- function(name, type, varnum, nullable, resrv, qryptr, arrptr) {
           if (is.na(varnum)) {
               if (type %in% c("CHAR", "ASCII", "UTF8")) {
+                  #message("Allocating with ", resrv, " and ", memory_budget)
                   buf <- libtiledb_query_buffer_var_char_alloc_direct(resrv, memory_budget, nullable)
                   qryptr <- libtiledb_query_set_buffer_var_char(qryptr, name, buf)
                   buf

R/Utils.R

@@ -51,7 +51,7 @@ save_return_as_preference <- function(value = c("asis", "array", "matrix", "data
     cfgdir <- tools::R_user_dir(packageName())
     if (!dir.exists(cfgdir)) dir.create(cfgdir)
     fname <- file.path(cfgdir, "config.dcf")
-    con <- file(fname, "w+")
+    con <- file(fname, "a+")
     cat("return_as:", value, "\n", file=con)
     close(con)
     set_return_as_preference(value)
@@ -107,7 +107,7 @@ set_return_as_preference <- function(value = c("asis", "array", "matrix", "data.
 ##'
 ##' Note that this memory budget (currently, at least) applies only to
 ##' character columns.  A fallback value of 50mb is used if no user
-##' value is set.
+##' value is set, and is used for all columns (rather than per column).
 ##'
 ##' @note This function requires R version 4.0.0 or later to utilise the per-user
 ##' config directory accessor function. For older R versions, a fallback from the
@@ -124,7 +124,7 @@ save_allocation_size_preference <- function(value) {
     cfgdir <- tools::R_user_dir(packageName())
     if (!dir.exists(cfgdir)) dir.create(cfgdir)
     fname <- file.path(cfgdir, "config.dcf")
-    con <- file(fname, "w+")
+    con <- file(fname, "a+")
     cat("allocation_size:", value, "\n", file=con)
     close(con)
     set_allocation_size_preference(value)