From 01e9684c3e19a1edbe362254a1b59d6747c0891f Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Sat, 15 Jan 2022 09:29:26 -0600 Subject: [PATCH 1/3] use append, not write, to alter config file --- R/Utils.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/Utils.R b/R/Utils.R index efd2fd3fec..eed7254608 100644 --- a/R/Utils.R +++ b/R/Utils.R @@ -51,7 +51,7 @@ save_return_as_preference <- function(value = c("asis", "array", "matrix", "data cfgdir <- tools::R_user_dir(packageName()) if (!dir.exists(cfgdir)) dir.create(cfgdir) fname <- file.path(cfgdir, "config.dcf") - con <- file(fname, "w+") + con <- file(fname, "a+") cat("return_as:", value, "\n", file=con) close(con) set_return_as_preference(value) @@ -124,7 +124,7 @@ save_allocation_size_preference <- function(value) { cfgdir <- tools::R_user_dir(packageName()) if (!dir.exists(cfgdir)) dir.create(cfgdir) fname <- file.path(cfgdir, "config.dcf") - con <- file(fname, "w+") + con <- file(fname, "a+") cat("allocation_size:", value, "\n", file=con) close(con) set_allocation_size_preference(value) From 7384bfb507e2df8fa94a8258a13d41a32287c16d Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Sat, 15 Jan 2022 09:29:47 -0600 Subject: [PATCH 2/3] make memory budget per column --- R/TileDBArray.R | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/R/TileDBArray.R b/R/TileDBArray.R index 05c9bd28cb..2aeebeafd1 100644 --- a/R/TileDBArray.R +++ b/R/TileDBArray.R @@ -519,8 +519,9 @@ setMethod("[", "tiledb_array", ## A preference can be set in a local per-user configuration file; if no value ## is set a fallback from the TileDB config object is used. Note that this memory - ## budget (currently, at least) applies only to character columns. - memory_budget <- get_allocation_size_preference() + ## budget (currently, at least) applies only to character columns. We scale the total + ## budget by the number of variable sized column (where 'varnum' is NA) + memory_budget <- get_allocation_size_preference() / max(1, sum(is.na(allvarnum)), na.rm=TRUE) if (length(enckey) > 0) { if (length(tstamp) > 0) { @@ -688,6 +689,7 @@ setMethod("[", "tiledb_array", getBuffer <- function(name, type, varnum, nullable, resrv, qryptr, arrptr) { if (is.na(varnum)) { if (type %in% c("CHAR", "ASCII", "UTF8")) { + #message("Allocating with ", resrv, " and ", memory_budget) buf <- libtiledb_query_buffer_var_char_alloc_direct(resrv, memory_budget, nullable) qryptr <- libtiledb_query_set_buffer_var_char(qryptr, name, buf) buf From 4624fc6e7127bccc7edeaa9333247985e162d086 Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Sat, 15 Jan 2022 09:31:43 -0600 Subject: [PATCH 3/3] update documentation --- R/Utils.R | 2 +- man/save_allocation_size_preference.Rd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/Utils.R b/R/Utils.R index eed7254608..62a34bd69d 100644 --- a/R/Utils.R +++ b/R/Utils.R @@ -107,7 +107,7 @@ set_return_as_preference <- function(value = c("asis", "array", "matrix", "data. ##' ##' Note that this memory budget (currently, at least) applies only to ##' character columns. A fallback value of 50mb is used if no user -##' value is set. +##' value is set, and is used for all columns (rather than per column). ##' ##' @note This function requires R version 4.0.0 or later to utilise the per-user ##' config directory accessor function. For older R versions, a fallback from the diff --git a/man/save_allocation_size_preference.Rd b/man/save_allocation_size_preference.Rd index 9a8eaa5aaa..765bd5e823 100644 --- a/man/save_allocation_size_preference.Rd +++ b/man/save_allocation_size_preference.Rd @@ -46,7 +46,7 @@ have no effect on standard allocation sizes. Note that this memory budget (currently, at least) applies only to character columns. A fallback value of 50mb is used if no user -value is set. +value is set, and is used for all columns (rather than per column). } \note{ This function requires R version 4.0.0 or later to utilise the per-user