From a40da1080322570da828b949b8f57dc2cea0ba18 Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Tue, 9 Nov 2021 20:45:51 -0600 Subject: [PATCH 1/4] remove max_buffer_elements calls in older tiledb_{dense,sparse} --- tools/tiledbVersion.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/tiledbVersion.txt b/tools/tiledbVersion.txt index 6338227295..d702730139 100644 --- a/tools/tiledbVersion.txt +++ b/tools/tiledbVersion.txt @@ -1,2 +1,2 @@ -version: 2.4.3 -sha: 5cb0824 +version: 2.5.0-rc0 +sha: 08cca1a From 84fc3483a9f6baae7ce6f77e7a90daf14e69f0ac Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Wed, 10 Nov 2021 07:48:09 -0600 Subject: [PATCH 2/4] remove max_buffer_elements calls in older tiledb_{dense,sparse} --- R/DenseArray.R | 7 +--- R/RcppExports.R | 12 ------ R/SparseArray.R | 8 ++-- src/RcppExports.cpp | 43 --------------------- src/deprecation.cpp | 85 ----------------------------------------- tools/tiledbVersion.txt | 4 +- 6 files changed, 7 insertions(+), 152 deletions(-) diff --git a/R/DenseArray.R b/R/DenseArray.R index 5799af71ad..81a194064f 100644 --- a/R/DenseArray.R +++ b/R/DenseArray.R @@ -213,8 +213,7 @@ attribute_buffers <- function(array, sch, dom, sub, selected) { # first alloc coordinate buffer if we are returning a data.frame if(array@as.data.frame) { - ncells_coords <- libtiledb_array_max_buffer_elements_with_type(array@ptr, sub, - libtiledb_coords(), domaintype) + ncells_coords <- 2*ncells if (is.integral(dom)) { attributes[["coords"]] <- integer(length = ncells_coords) } else { @@ -234,10 +233,6 @@ attribute_buffers <- function(array, sch, dom, sub, selected) { type <- tiledb_datatype_R_type(dtype) datatype <- libtiledb_attribute_get_type(attr@ptr) #cat("dtype:", dtype, " type:", type, " datatype:", datatype, "\n", sep="") - ## If getting it as a dataframe we need to use max buffer elements to get proper buffer size - if (array@as.data.frame) { - ncells <- libtiledb_array_max_buffer_elements_with_type(array@ptr, sub, aname, domaintype) - } if (type %in% c("integer", "double")) { buff <- vector(mode = type, length = ncells) } else if (dtype %in% c("CHAR")) { # TODO: add other char and date types diff --git a/R/RcppExports.R b/R/RcppExports.R index fd6cacd267..3dfbf83093 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -33,18 +33,6 @@ libtiledb_coords <- function() { .Call(`_tiledb_libtiledb_coords`) } -libtiledb_array_max_buffer_elements_with_type <- function(array, subarray, attribute, typestr) { - .Call(`_tiledb_libtiledb_array_max_buffer_elements_with_type`, array, subarray, attribute, typestr) -} - -libtiledb_array_max_buffer_elements <- function(array, subarray, attribute) { - .Call(`_tiledb_libtiledb_array_max_buffer_elements`, array, subarray, attribute) -} - -libtiledb_array_max_buffer_elements_vec <- function(array, subarray, attribute) { - .Call(`_tiledb_libtiledb_array_max_buffer_elements_vec`, array, subarray, attribute) -} - tiledb_datatype_R_type <- function(datatype) { .Call(`_tiledb_tiledb_datatype_R_type`, datatype) } diff --git a/R/SparseArray.R b/R/SparseArray.R index e89ac3560a..32df02b705 100644 --- a/R/SparseArray.R +++ b/R/SparseArray.R @@ -96,7 +96,7 @@ setMethod("schema", "tiledb_sparse", function(object, ...) { return(tiledb_array_schema.from_ptr(schema_xptr)) }) -sparse_attribute_buffers <- function(array, sch, dom, sub, selected) { +sparse_attribute_buffers <- function(array, sch, dom, sub, selected, ncells=1000) { stopifnot(is(sch, "tiledb_array_schema")) stopifnot(is(dom, "tiledb_domain")) #domaintype <- libtiledb_domain_get_type(dom@ptr) @@ -105,8 +105,8 @@ sparse_attribute_buffers <- function(array, sch, dom, sub, selected) { attributes <- list() # first alloc coordinate buffer #print(domaintype) - ncells <- libtiledb_array_max_buffer_elements_with_type(array@ptr, sub, - libtiledb_coords(), domaintype[1]) + #ncells <- libtiledb_array_max_buffer_elements_with_type(array@ptr, sub, + # libtiledb_coords(), domaintype[1]) attributes[["coords"]] <- libtiledb_query_buffer_alloc_ptr(array@ptr, domaintype[1], ncells) attrs <- tiledb::attrs(sch) if (length(selected) == 0) { # no selection given -> use all @@ -122,7 +122,7 @@ sparse_attribute_buffers <- function(array, sch, dom, sub, selected) { type <- tiledb_datatype_R_type(dtype) datatype <- libtiledb_attribute_get_type(attr@ptr) #cat("dtype:", dtype, " type:", type, " datatype:", datatype, "\n", sep="") - ncells <- libtiledb_array_max_buffer_elements_with_type(array@ptr, sub, aname, domaintype[1]) + #ncells <- libtiledb_array_max_buffer_elements_with_type(array@ptr, sub, aname, domaintype[1]) if (dtype %in% c("CHAR")) { # TODO: add other char and date types #buff <- libtiledb_query_buffer_var_char_alloc(array@ptr, sub, aname) buff <- libtiledb_query_buffer_var_char_alloc_direct(ncells, ncells*8, FALSE, sub[4]-sub[3]+1) diff --git a/src/RcppExports.cpp b/src/RcppExports.cpp index be81e207e6..e90714a6dc 100644 --- a/src/RcppExports.cpp +++ b/src/RcppExports.cpp @@ -102,46 +102,6 @@ BEGIN_RCPP return rcpp_result_gen; END_RCPP } -// libtiledb_array_max_buffer_elements_with_type -R_xlen_t libtiledb_array_max_buffer_elements_with_type(XPtr array, SEXP subarray, std::string attribute, std::string typestr); -RcppExport SEXP _tiledb_libtiledb_array_max_buffer_elements_with_type(SEXP arraySEXP, SEXP subarraySEXP, SEXP attributeSEXP, SEXP typestrSEXP) { -BEGIN_RCPP - Rcpp::RObject rcpp_result_gen; - Rcpp::RNGScope rcpp_rngScope_gen; - Rcpp::traits::input_parameter< XPtr >::type array(arraySEXP); - Rcpp::traits::input_parameter< SEXP >::type subarray(subarraySEXP); - Rcpp::traits::input_parameter< std::string >::type attribute(attributeSEXP); - Rcpp::traits::input_parameter< std::string >::type typestr(typestrSEXP); - rcpp_result_gen = Rcpp::wrap(libtiledb_array_max_buffer_elements_with_type(array, subarray, attribute, typestr)); - return rcpp_result_gen; -END_RCPP -} -// libtiledb_array_max_buffer_elements -R_xlen_t libtiledb_array_max_buffer_elements(XPtr array, SEXP subarray, std::string attribute); -RcppExport SEXP _tiledb_libtiledb_array_max_buffer_elements(SEXP arraySEXP, SEXP subarraySEXP, SEXP attributeSEXP) { -BEGIN_RCPP - Rcpp::RObject rcpp_result_gen; - Rcpp::RNGScope rcpp_rngScope_gen; - Rcpp::traits::input_parameter< XPtr >::type array(arraySEXP); - Rcpp::traits::input_parameter< SEXP >::type subarray(subarraySEXP); - Rcpp::traits::input_parameter< std::string >::type attribute(attributeSEXP); - rcpp_result_gen = Rcpp::wrap(libtiledb_array_max_buffer_elements(array, subarray, attribute)); - return rcpp_result_gen; -END_RCPP -} -// libtiledb_array_max_buffer_elements_vec -NumericVector libtiledb_array_max_buffer_elements_vec(XPtr array, SEXP subarray, std::string attribute); -RcppExport SEXP _tiledb_libtiledb_array_max_buffer_elements_vec(SEXP arraySEXP, SEXP subarraySEXP, SEXP attributeSEXP) { -BEGIN_RCPP - Rcpp::RObject rcpp_result_gen; - Rcpp::RNGScope rcpp_rngScope_gen; - Rcpp::traits::input_parameter< XPtr >::type array(arraySEXP); - Rcpp::traits::input_parameter< SEXP >::type subarray(subarraySEXP); - Rcpp::traits::input_parameter< std::string >::type attribute(attributeSEXP); - rcpp_result_gen = Rcpp::wrap(libtiledb_array_max_buffer_elements_vec(array, subarray, attribute)); - return rcpp_result_gen; -END_RCPP -} // tiledb_datatype_R_type std::string tiledb_datatype_R_type(std::string datatype); RcppExport SEXP _tiledb_tiledb_datatype_R_type(SEXP datatypeSEXP) { @@ -2724,9 +2684,6 @@ static const R_CallMethodDef CallEntries[] = { {"_tiledb_libtiledb_query_import_buffer", (DL_FUNC) &_tiledb_libtiledb_query_import_buffer, 4}, {"_tiledb_libtiledb_query_set_coordinates", (DL_FUNC) &_tiledb_libtiledb_query_set_coordinates, 3}, {"_tiledb_libtiledb_coords", (DL_FUNC) &_tiledb_libtiledb_coords, 0}, - {"_tiledb_libtiledb_array_max_buffer_elements_with_type", (DL_FUNC) &_tiledb_libtiledb_array_max_buffer_elements_with_type, 4}, - {"_tiledb_libtiledb_array_max_buffer_elements", (DL_FUNC) &_tiledb_libtiledb_array_max_buffer_elements, 3}, - {"_tiledb_libtiledb_array_max_buffer_elements_vec", (DL_FUNC) &_tiledb_libtiledb_array_max_buffer_elements_vec, 3}, {"_tiledb_tiledb_datatype_R_type", (DL_FUNC) &_tiledb_tiledb_datatype_R_type, 1}, {"_tiledb_libtiledb_version", (DL_FUNC) &_tiledb_libtiledb_version, 0}, {"_tiledb_libtiledb_ctx", (DL_FUNC) &_tiledb_libtiledb_ctx, 1}, diff --git a/src/deprecation.cpp b/src/deprecation.cpp index 77b513fed7..69f3b2ee66 100644 --- a/src/deprecation.cpp +++ b/src/deprecation.cpp @@ -34,17 +34,6 @@ using namespace Rcpp; -// [[Rcpp::plugins(cpp11)]] - - -// declarations needed -XPtr libtiledb_array_get_schema(XPtr array); -XPtr libtiledb_array_schema_get_domain(XPtr schema); -XPtr libtiledb_array_schema_get_attribute_from_name(XPtr schema, std::string name); -std::string libtiledb_attribute_get_type(XPtr attr); - - - // [[Rcpp::export]] XPtr libtiledb_query_set_coordinates(XPtr query, SEXP coords, @@ -78,77 +67,3 @@ XPtr libtiledb_query_set_coordinates(XPtr query, std::string libtiledb_coords() { return tiledb_coords(); } - - -// using domain type information -// [[Rcpp::export]] -R_xlen_t libtiledb_array_max_buffer_elements_with_type(XPtr array, - SEXP subarray, - std::string attribute, - std::string typestr) { - if (typestr == "INT32") { - auto sub = as>(subarray); - auto max_elements = array->max_buffer_elements(sub); - return max_elements[attribute].second; - } else if (typestr == "FLOAT64") { - auto sub = as>(subarray); - auto max_elements = array->max_buffer_elements(sub); - return max_elements[attribute].second; - } else if (typestr == "INT64" || - typestr == "UINT64" || - typestr == "UINT32" || - typestr == "DATETIME_DAY" || - typestr == "DATETIME_HR" || - typestr == "DATETIME_MIN" || - typestr == "DATETIME_SEC" || - typestr == "DATETIME_MS" || - typestr == "DATETIME_US" || - typestr == "DATETIME_NS") { - NumericVector svec(subarray); - std::vector v(svec.size()); - for (int i=0; i(svec[i]); - } - auto max_elements = array->max_buffer_elements(v); - return max_elements[attribute].second; - } else { - Rcpp::stop("Invalid subarray buffer type '%s' for domain: '%s'", - typestr.c_str(), Rcpp::type2name(subarray)); - } -} - - -// [[Rcpp::export]] -R_xlen_t libtiledb_array_max_buffer_elements(XPtr array, - SEXP subarray, - std::string attribute) { - if (TYPEOF(subarray) == INTSXP) { - auto sub = as>(subarray); - auto max_elements = array->max_buffer_elements(sub); - return max_elements[attribute].second; - } else if (TYPEOF(subarray) == REALSXP) { - auto sub = as>(subarray); - auto max_elements = array->max_buffer_elements(sub); - return max_elements[attribute].second; - } else { - Rcpp::stop("Invalid subarray buffer type for domain: '%s'", Rcpp::type2name(subarray)); - } -} - - -// [[Rcpp::export]] -NumericVector libtiledb_array_max_buffer_elements_vec(XPtr array, - SEXP subarray, - std::string attribute) { - if (TYPEOF(subarray) == INTSXP) { - auto sub = as>(subarray); - auto max_elements = array->max_buffer_elements(sub); - return NumericVector::create(max_elements[attribute].first, max_elements[attribute].second); - } else if (TYPEOF(subarray) == REALSXP) { - auto sub = as>(subarray); - auto max_elements = array->max_buffer_elements(sub); - return NumericVector::create(max_elements[attribute].first, max_elements[attribute].second); - } else { - Rcpp::stop("Invalid subarray buffer type for domain %s", Rcpp::type2name(subarray)); - } -} diff --git a/tools/tiledbVersion.txt b/tools/tiledbVersion.txt index d702730139..6338227295 100644 --- a/tools/tiledbVersion.txt +++ b/tools/tiledbVersion.txt @@ -1,2 +1,2 @@ -version: 2.5.0-rc0 -sha: 08cca1a +version: 2.4.3 +sha: 5cb0824 From 34f66920a4eedab74100a0debefdfd5efb204345 Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Wed, 10 Nov 2021 08:02:35 -0600 Subject: [PATCH 3/4] remove some commented-out code --- R/SparseArray.R | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/R/SparseArray.R b/R/SparseArray.R index 32df02b705..59cdf25d9c 100644 --- a/R/SparseArray.R +++ b/R/SparseArray.R @@ -1,6 +1,6 @@ # MIT License # -# Copyright (c) 2017-2020 TileDB Inc. +# Copyright (c) 2017-2021 TileDB Inc. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -104,9 +104,6 @@ sparse_attribute_buffers <- function(array, sch, dom, sub, selected, ncells=1000 libtiledb_dim_get_datatype) attributes <- list() # first alloc coordinate buffer - #print(domaintype) - #ncells <- libtiledb_array_max_buffer_elements_with_type(array@ptr, sub, - # libtiledb_coords(), domaintype[1]) attributes[["coords"]] <- libtiledb_query_buffer_alloc_ptr(array@ptr, domaintype[1], ncells) attrs <- tiledb::attrs(sch) if (length(selected) == 0) { # no selection given -> use all @@ -121,10 +118,7 @@ sparse_attribute_buffers <- function(array, sch, dom, sub, selected, ncells=1000 dtype <- tiledb::datatype(attr) type <- tiledb_datatype_R_type(dtype) datatype <- libtiledb_attribute_get_type(attr@ptr) - #cat("dtype:", dtype, " type:", type, " datatype:", datatype, "\n", sep="") - #ncells <- libtiledb_array_max_buffer_elements_with_type(array@ptr, sub, aname, domaintype[1]) if (dtype %in% c("CHAR")) { # TODO: add other char and date types - #buff <- libtiledb_query_buffer_var_char_alloc(array@ptr, sub, aname) buff <- libtiledb_query_buffer_var_char_alloc_direct(ncells, ncells*8, FALSE, sub[4]-sub[3]+1) } else if (datatype %in% c("DATETIME_DAY", "DATETIME_SEC", "DATETIME_MS", "DATETIME_US", "DATETIME_NS")) { From 6e2aec76244936f8ddf6f8550811f90e7a4cf364 Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Wed, 10 Nov 2021 08:15:50 -0600 Subject: [PATCH 4/4] extra buffer time for macOS which is not as quick as the other kids --- inst/tinytest/test_fragmentinfo.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/inst/tinytest/test_fragmentinfo.R b/inst/tinytest/test_fragmentinfo.R index 1061ac7bda..87c1bb7217 100644 --- a/inst/tinytest/test_fragmentinfo.R +++ b/inst/tinytest/test_fragmentinfo.R @@ -5,6 +5,7 @@ if (tiledb_version(TRUE) < "2.2.0") exit_file("Needs TileDB 2.2.* or later") isOldWindows <- Sys.info()[["sysname"]] == "Windows" && grepl('Windows Server 2008', osVersion) if (isOldWindows) exit_file("skip this file on old Windows releases") +isMacOS <- (Sys.info()['sysname'] == "Darwin") ctx <- tiledb_ctx(limitTileDBCores()) @@ -47,7 +48,7 @@ rng <- tiledb_fragment_info_get_timestamp_range(fraginf, 0) expect_true(inherits(rng, "POSIXt")) expect_equal(length(rng), 2) expect_equal(as.Date(rng[1]), as.Date(as.POSIXlt(Sys.time(), tz="UTC"))) # very coarse :) -expect_true(as.numeric(difftime(Sys.time(), rng[1], "secs")) < 0.5) # at most 0.5 sec after array creation +expect_true(as.numeric(difftime(Sys.time(), rng[1], "secs")) < 0.5 + 1.0*isMacOS) # at most 0.5 sec after array creation (plus extra for maxOS which failed here randomly) expect_equal(tiledb_fragment_info_get_cell_num(fraginf, 0), 10) expect_true(tiledb_fragment_info_get_version(fraginf, 0) > 5) # we may test with older core libs