From 37b2c0c2bfe8aa77efc037a98b69e102552ea96d Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Mon, 28 Nov 2022 10:17:30 -0600 Subject: [PATCH 1/7] Default sparse read via tiledb_array to UNORDERED --- R/TileDBArray.R | 4 +++- inst/tinytest/test_matrix.R | 5 ++--- inst/tinytest/test_tiledbarray.R | 24 ++++++++++++------------ 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/R/TileDBArray.R b/R/TileDBArray.R index 1ef067cb04..e75003b4a0 100644 --- a/R/TileDBArray.R +++ b/R/TileDBArray.R @@ -592,7 +592,9 @@ setMethod("[", "tiledb_array", nonemptydom <- mapply(getDomain, dimnames, dimtypes, SIMPLIFY=FALSE) ## open query qryptr <- libtiledb_query(ctx@ptr, arrptr, "READ") - if (length(layout) > 0) libtiledb_query_set_layout(qryptr, layout) + qryptr <- libtiledb_query_set_layout(qryptr, + if (length(layout) > 0) layout + else { if (sparse) "UNORDERED" else "COL_MAJOR" }) ## ranges seem to interfere with the byte/element adjustment below so set up toggle rangeunset <- TRUE diff --git a/inst/tinytest/test_matrix.R b/inst/tinytest/test_matrix.R index 63765796ee..3376e2d418 100644 --- a/inst/tinytest/test_matrix.R +++ b/inst/tinytest/test_matrix.R @@ -8,11 +8,11 @@ if (isOldWindows) exit_file("skip this file on old Windows releases") ctx <- tiledb_ctx(limitTileDBCores()) uri <- tempfile() -M <- matrix(1:12, 3, 4, dimnames=list(LETTERS[1:3], letters[1:4])) +M <- matrix(1:16, 4, 4, dimnames=list(LETTERS[1:4], letters[1:4])) fromMatrix(M, uri) M2 <- toMatrix(uri) -expect_equivalent(M, M2) +expect_equivalent(M, t(M2)) # because we now default to UNORDERED we need to transpose uri <- tempfile() @@ -23,7 +23,6 @@ M2 <- toMatrix(uri) expect_equivalent(M, M2) - uri <- tempfile() M <- matrix(sqrt(1:20), 5, 4) # matrix without rownames, float values fromMatrix(M, uri) diff --git a/inst/tinytest/test_tiledbarray.R b/inst/tinytest/test_tiledbarray.R index 471a4641d0..d83e8d9b5c 100644 --- a/inst/tinytest/test_tiledbarray.R +++ b/inst/tinytest/test_tiledbarray.R @@ -599,7 +599,7 @@ if (requireNamespace("bit64", quietly=TRUE)) { ## or with indices #A[rep(1:4,each=4), rep(1:4,4)] <- data - A <- tiledb_array(uri = tmp, as.data.frame=TRUE) + A <- tiledb_array(uri = tmp, as.data.frame=TRUE, query_layout="ROW_MAJOR") newdata <- A[1:2, 2:3] expect_equal(newdata[,"a"], c(2L, 3L, 6L, 7L)) expect_equal(newdata[,"rows"], c(1L, 1L, 2L, 2L)) @@ -630,7 +630,7 @@ if (requireNamespace("bit64", quietly=TRUE)) { ## or with indices #A[rep(1:4,each=4), rep(1:4,4)] <- data - A <- tiledb_array(uri = tmp, as.data.frame=TRUE) + A <- tiledb_array(uri = tmp, as.data.frame=TRUE, query_layout="ROW_MAJOR") newdata <- A[1:2, 2:3] expect_equal(newdata[,"a"], c(2L, 3L, 6L, 7L)) expect_equal(newdata[,"rows"], c(1L, 1L, 2L, 2L)) @@ -661,7 +661,7 @@ if (requireNamespace("bit64", quietly=TRUE)) { ## or with indices #A[rep(1:4,each=4), rep(1:4,4)] <- data - A <- tiledb_array(uri = tmp, as.data.frame=TRUE) + A <- tiledb_array(uri = tmp, as.data.frame=TRUE, query_layout="ROW_MAJOR") newdata <- A[1:2, 2:3] expect_equal(newdata[,"a"], c(2L, 3L, 6L, 7L)) expect_equal(newdata[,"rows"], c(1L, 1L, 2L, 2L)) @@ -692,7 +692,7 @@ if (requireNamespace("bit64", quietly=TRUE)) { ## or with indices #A[rep(1:4,each=4), rep(1:4,4)] <- data - A <- tiledb_array(uri = tmp, as.data.frame=TRUE) + A <- tiledb_array(uri = tmp, as.data.frame=TRUE, query_layout="ROW_MAJOR") newdata <- A[1:2, 2:3] expect_equal(newdata[,"a"], c(2L, 3L, 6L, 7L)) expect_equal(newdata[,"rows"], c(1L, 1L, 2L, 2L)) @@ -723,7 +723,7 @@ if (requireNamespace("bit64", quietly=TRUE)) { ## or with indices #A[rep(1:4,each=4), rep(1:4,4)] <- data - A <- tiledb_array(uri = tmp, as.data.frame=TRUE) + A <- tiledb_array(uri = tmp, as.data.frame=TRUE, query_layout="ROW_MAJOR") newdata <- A[1:2, 2:3] expect_equal(newdata[,"a"], c(2L, 3L, 6L, 7L)) expect_equal(newdata[,"rows"], c(1L, 1L, 2L, 2L)) @@ -754,7 +754,7 @@ if (requireNamespace("bit64", quietly=TRUE)) { ## or with indices #A[rep(1:4,each=4), rep(1:4,4)] <- data - A <- tiledb_array(uri = tmp, as.data.frame=TRUE) + A <- tiledb_array(uri = tmp, as.data.frame=TRUE, query_layout="ROW_MAJOR") newdata <- A[1:2, 2:3] expect_equal(newdata[,"a"], c(2L, 3L, 6L, 7L)) expect_equal(newdata[,"rows"], c(1L, 1L, 2L, 2L)) @@ -785,7 +785,7 @@ if (requireNamespace("bit64", quietly=TRUE)) { ## or with indices #A[as.integer64(rep(1:4,each=4)), as.integer64(rep(1:4,4))] <- data - A <- tiledb_array(uri = tmp, as.data.frame=TRUE) + A <- tiledb_array(uri = tmp, as.data.frame=TRUE, query_layout="ROW_MAJOR") newdata <- A[as.integer64(1:2), as.integer64(2:3)] expect_equal(newdata[,"a"], c(2L, 3L, 6L, 7L)) expect_equal(newdata[,"rows"], as.integer64(c(1L, 1L, 2L, 2L))) @@ -817,7 +817,7 @@ if (requireNamespace("bit64", quietly=TRUE)) { ## or with indices #A[as.integer64(rep(1:4,each=4)), as.integer64(rep(1:4,4))] <- data - A <- tiledb_array(uri = tmp, as.data.frame=TRUE) + A <- tiledb_array(uri = tmp, as.data.frame=TRUE, query_layout="ROW_MAJOR") newdata <- A[as.integer64(1:2), as.integer64(2:3)] expect_equal(newdata[,"a"], c(2L, 3L, 6L, 7L)) expect_equal(newdata[,"rows"], c(1L, 1L, 2L, 2L)) @@ -868,7 +868,7 @@ if (requireNamespace("bit64", quietly=TRUE)) { ## or with indices #A[rep(1:4,each=4), rep(1:4,4)] <- data - A <- tiledb_array(uri = tmp, as.data.frame=TRUE) + A <- tiledb_array(uri = tmp, as.data.frame=TRUE, query_layout="ROW_MAJOR") newdata <- A[1:2, 2:3] expect_equal(newdata[,"rows"], c(1L, 1L, 2L, 2L)) expect_equal(newdata[,"cols"], c(2L, 3L, 2L, 3L)) @@ -924,7 +924,7 @@ if (requireNamespace("bit64", quietly=TRUE)) { ## or with indices A[rep(1:4,each=4), rep(1:4,4)] <- data - A <- tiledb_array(uri = tmp, as.data.frame=TRUE) + A <- tiledb_array(uri = tmp, as.data.frame=TRUE, query_layout="ROW_MAJOR") newdata <- A[1:2, 2:3] expect_equal(newdata[,"rows"], c(1L, 1L, 2L, 2L)) expect_equal(newdata[,"cols"], c(2L, 3L, 2L, 3L)) @@ -1351,7 +1351,7 @@ data <- array(1:64, dim = c(4,4,4)) A <- tiledb_array(uri = uri) A[] <- data -A <- tiledb_array(uri = uri, return_as="data.frame") +A <- tiledb_array(uri = uri, return_as="data.frame", query_layout="ROW_MAJOR") res <- A[2,2,2] expect_equal(res[, "a", drop=TRUE], 22) res <- A[2,2:3,2] @@ -1421,7 +1421,7 @@ if (v[["major"]] == 2L && v[["minor"]] %in% c(4L, 10L)) exit_file("Skip remainde ## check for incomplete status on unsuccessful query set_allocation_size_preference(256) # too low for penguins to return something -array <- tiledb_array(uri, as.data.frame=TRUE) +array <- tiledb_array(uri, as.data.frame=TRUE, query_layout="ROW_MAJOR") expect_warning(res <- array[]) # warning emitted expect_equal(attr(res, "query_status"), "INCOMPLETE") # and query status reported From 34e915099dddf6749b2b70e92b80cd066a7b0698 Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Mon, 28 Nov 2022 14:23:07 -0600 Subject: [PATCH 2/7] Use nchar(layout) not length, and wrap in isTRUE for comparison --- R/TileDBArray.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/TileDBArray.R b/R/TileDBArray.R index e75003b4a0..ca4af73897 100644 --- a/R/TileDBArray.R +++ b/R/TileDBArray.R @@ -593,7 +593,7 @@ setMethod("[", "tiledb_array", ## open query qryptr <- libtiledb_query(ctx@ptr, arrptr, "READ") qryptr <- libtiledb_query_set_layout(qryptr, - if (length(layout) > 0) layout + if (isTRUE(nchar(layout) > 0)) layout else { if (sparse) "UNORDERED" else "COL_MAJOR" }) ## ranges seem to interfere with the byte/element adjustment below so set up toggle @@ -1157,7 +1157,7 @@ setMethod("[<-", "tiledb_array", qryptr <- libtiledb_query(ctx@ptr, arrptr, "WRITE") qryptr <- libtiledb_query_set_layout(qryptr, - if (length(layout) > 0) layout + if (isTRUE(nchar(layout) > 0)) layout else { if (sparse) "UNORDERED" else "COL_MAJOR" }) buflist <- vector(mode="list", length=nc) From d5abc3afc11d7f478756098943b5040004d5c44d Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Fri, 18 Nov 2022 09:42:42 -0600 Subject: [PATCH 3/7] Update NEWS.md for #483, #484, #485, and #487 --- NEWS.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 2bd0649f58..cebd4cd1df 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,7 +4,7 @@ * Support for testing group URIs on being relative has been added (#478) -* Logging support at the R and C++ level has been added (#479) +* Logging support at the R and C++ level has been added (#479, #487) * Use of TileDB Embedded was upgraded to release 2.12.1, and 2.12.2 (#480, #481) @@ -12,10 +12,16 @@ * Accomodate possible zero sized allocation estimates for attributes (#482) +* Detect missing columns in a write-attempt with partial data (#483) + ## Build and Test Systems * Update check-out action to version three suppressing a warning (#477) +* Code Coverage reports are now generated and available at codecov.io (#484) + +* Small internal changes renaming two files and conditioning tests under two older releases (#485) + # tiledb 0.16.0 From 8bf517654b483b9397c3b4f24abb1ad7096a9228 Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Mon, 28 Nov 2022 14:29:11 -0600 Subject: [PATCH 4/7] Update NEWS for #488 [ci skip] --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index cebd4cd1df..d36ef64f5b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -8,6 +8,8 @@ * Use of TileDB Embedded was upgraded to release 2.12.1, and 2.12.2 (#480, #481) +* Sparse array queries via tiledb_array and '[]' access use an UNORDERED query layout (#488) + ## Bug Fixes * Accomodate possible zero sized allocation estimates for attributes (#482) From 6373af06f1a60f89ce35e5deae1d0d037467c241 Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Mon, 28 Nov 2022 15:34:08 -0600 Subject: [PATCH 5/7] Roll micro release to 0.16.0.3 [ci skip] --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 0e14922e7a..c64a096566 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: tiledb Type: Package -Version: 0.16.0.2 +Version: 0.16.0.3 Title: Universal Storage Engine for Sparse and Dense Multidimensional Arrays Authors@R: c(person("TileDB, Inc.", role = c("aut", "cph")), person("Dirk", "Eddelbuettel", email = "dirk@tiledb.com", role = "cre")) From c06ed3dd2a1c9678fa09942daf9ce68b0452ae0e Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Mon, 28 Nov 2022 17:20:07 -0600 Subject: [PATCH 6/7] Add codecov.yml to .Rbuildignore --- .Rbuildignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.Rbuildignore b/.Rbuildignore index 036e0bb4a7..18cd4ec347 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -25,3 +25,4 @@ ^vignettes/Makefile ^inst/tinytest/test_timetravel_extra.R ^inst/tinytest/test_tiledbarray_extra.R +^codecov.yml From 782d5a9ff1e50c110b106264d765a27b9abefaf8 Mon Sep 17 00:00:00 2001 From: Dirk Eddelbuettel Date: Mon, 28 Nov 2022 17:51:23 -0600 Subject: [PATCH 7/7] Tighten one version-based test exclusion --- inst/tinytest/test_arrayschemaevolution.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inst/tinytest/test_arrayschemaevolution.R b/inst/tinytest/test_arrayschemaevolution.R index 65c1f7b6ed..a360c35e3d 100644 --- a/inst/tinytest/test_arrayschemaevolution.R +++ b/inst/tinytest/test_arrayschemaevolution.R @@ -6,7 +6,7 @@ if (Sys.info()[["sysname"]] == "Windows") exit_file("Skip on Windows") ctx <- tiledb_ctx(limitTileDBCores()) -if (tiledb_version(TRUE) < "2.4.0") exit_file("Needs TileDB 2.4.* or later") +if (tiledb_version(TRUE) < "2.11.0") exit_file("Needs TileDB 2.11.* or later") df <- data.frame(key=letters[1:10], val=1:10)