Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Map (u)int64 indices and restore third dim when indexing #365

Merged
merged 4 commits into from Feb 3, 2022
Merged

Map (u)int64 indices and restore third dim when indexing #365

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e218ccc
extend int64 mapping for dimension from datetimes to ints
eddelbuettel Feb 2, 2022
ec0a192
support third dimension in A[i,j,k] indexing
eddelbuettel Feb 2, 2022
c68ee79
tests for added functionality
eddelbuettel Feb 3, 2022
7d6e0ed
additional tests for i, j, k in isolation
eddelbuettel Feb 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
41 changes: 31 additions & 10 deletions R/TileDBArray.R
View file
Open in desktop
Expand Up @@ -420,9 +420,9 @@ setValidity("tiledb_array", function(object) {
## as.numeric(as.Date("2021-01-01")) yields 18628.
##
## We also convert the value to integer64 because that is the internal storage format
.mapDatetime2integer64 <- function(val, dtype) {
## in case it is not a datetime type, or already an int64, return unchanged
if (!grepl("^DATETIME_", dtype) || inherits(val, "integer64"))
.map2integer64 <- function(val, dtype) {
## in case it is not a (datetime or (u)int64) type), or already an int64, return unchanged
if ((!grepl("^DATETIME_", dtype) && !grepl("INT64$", dtype)) || inherits(val, "integer64"))
return(val)

val <- switch(dtype,
Expand All @@ -438,11 +438,12 @@ setValidity("tiledb_array", function(object) {
"DATETIME_NS" = as.numeric(val),
"DATETIME_PS" = as.numeric(val) * 1e3,
"DATETIME_FS" = as.numeric(val) * 1e6,
"DATETIME_AS" = as.numeric(val) * 1e9)
"DATETIME_AS" = as.numeric(val) * 1e9,
"UINT64" = val,
"INT64" = val)
bit64::as.integer64(val)
}


#' Returns a TileDB array, allowing for specific subset ranges.
#'
#' Heterogenous domains are supported, including timestamps and characters.
Expand All @@ -469,6 +470,16 @@ setMethod("[", "tiledb_array",
## add defaults
if (missing(i)) i <- NULL
ihnorton marked this conversation as resolved.
Show resolved Hide resolved
if (missing(j)) j <- NULL
k <- NULL

## deal with possible n-dim indexing
ndlist <- nd_index_from_syscall(sys.call(), parent.frame())
if (length(ndlist) >= 0) {
if (length(ndlist) >= 1 && !is.null(ndlist[[1]])) i <- ndlist[[1]]
if (length(ndlist) >= 2 && !is.null(ndlist[[2]])) j <- ndlist[[2]]
if (length(ndlist) >= 3 && !is.null(ndlist[[3]])) k <- ndlist[[3]]
if (length(ndlist) >= 4) message("Indices beyond the third dimension not supported in [i,j,k] form. Use selected_ranges().")
}

ctx <- x@ctx
uri <- x@uri
Expand Down Expand Up @@ -608,25 +619,35 @@ setMethod("[", "tiledb_array",
x@selected_ranges[[2]] <- j
}

if (!is.null(k)) {
if (!is.null(x@selected_ranges[[3]])) {
stop("Cannot set both 'k' and second element of 'selected_ranges'.", call. = FALSE)
}
x@selected_ranges[[3]] <- k
}
## (i,j,k) are now done and transferred to x@select_ranges


## if ranges selected, use those
for (k in seq_len(length(x@selected_ranges))) {
if (is.null(x@selected_ranges[[k]])) {
#cat("Adding null dim", k, "on", dimtypes[[k]], "\n")
vec <- .mapDatetime2integer64(nonemptydom[[k]], dimtypes[k])
#cat("Adding null dim", k, "on", dimtypes[k], "\n")
vec <- .map2integer64(nonemptydom[[k]], dimtypes[k])
if (vec[1] != 0 && vec[2] != 0) { # corner case of A[] on empty array
qryptr <- libtiledb_query_add_range_with_type(qryptr, k-1, dimtypes[k], vec[1], vec[2])
rangeunset <- FALSE
}
} else if (is.null(nrow(x@selected_ranges[[k]]))) {
#cat("Adding nrow null dim", k, "on", dimtypes[[k]], "\n")
#cat("Adding nrow null dim", k, "on", dimtypes[k], "\n")
vec <- x@selected_ranges[[k]]
vec <- .map2integer64(vec, dimtypes[k])
qryptr <- libtiledb_query_add_range_with_type(qryptr, k-1, dimtypes[k], min(vec), max(vec))
rangeunset <- FALSE
} else {
#cat("Adding non-zero dim", k, "on", dimtypes[[k]], "\n")
#cat("Adding non-zero dim", k, "on", dimtypes[k], "\n")
m <- x@selected_ranges[[k]]
for (i in seq_len(nrow(m))) {
vec <- .mapDatetime2integer64(c(m[i,1], m[i,2]), dimtypes[k])
vec <- .map2integer64(c(m[i,1], m[i,2]), dimtypes[k])
qryptr <- libtiledb_query_add_range_with_type(qryptr, k-1, dimtypes[k], vec[1], vec[2])
}
rangeunset <- FALSE
Expand Down
59 changes: 59 additions & 0 deletions inst/tinytest/test_tiledbarray.R
View file
Open in desktop
Expand Up @@ -1337,3 +1337,62 @@ expect_equal(dim(res), c(14,6))
expect_true(min(res$body_mass_g) >= 5500)
expect_true(min(res$bill_length_mm) > 50)
expect_equal(colnames(res), c("species", "island", "body_mass_g", "bill_length_mm", "year", "sex"))


## new 3d index, and int64 domain conversion
uri <- tempfile()
dom <- tiledb_domain(dims = c(tiledb_dim("rows", c(1L, 4L), 4L, "INT32"),
tiledb_dim("cols", c(1L, 4L), 4L, "INT32"),
tiledb_dim("depth", c(1L, 4L), 4L, "INT32")))
schema <- tiledb_array_schema(dom, attrs = c(tiledb_attr("a", type = "INT32")))
tiledb_array_create(uri, schema)
data <- array(1:64, dim = c(4,4,4))
A <- tiledb_array(uri = uri)
A[] <- data

A <- tiledb_array(uri = uri, return_as="data.frame")
res <- A[2,2,2]
expect_equal(res[, "a", drop=TRUE], 22)
res <- A[2,2:3,2]
expect_equal(res[, "a", drop=TRUE], c(22,26))
res <- A[2,]
expect_true(all(res[, "rows", drop=TRUE] == 2))
expect_equal(res[, "a", drop=TRUE], c(2L, 18L, 34L, 50L, 6L, 22L, 38L, 54L, 10L, 26L, 42L, 58L, 14L,
30L, 46L, 62L))
res <- A[,2]
expect_true(all(res[, "cols", drop=TRUE] == 2))
expect_equal(res[, "a", drop=TRUE], c(5L, 21L, 37L, 53L, 6L, 22L, 38L, 54L, 7L, 23L, 39L, 55L, 8L, 24L, 40L, 56L))
res <- A[,,2]
expect_true(all(res[, "depth", drop=TRUE] == 2))
expect_equal(res[, "a", drop=TRUE], c(17L, 21L, 25L, 29L, 18L, 22L, 26L, 30L, 19L, 23L, 27L, 31L, 20L, 24L, 28L, 32L))
selected_ranges(A) <- list(cbind(2,2), cbind(2,2), cbind(2,2))
res <- A[]
expect_equal(res[, "a", drop=TRUE], 22)
selected_ranges(A) <- list(cbind(2,2), cbind(2,3), cbind(2,2))
res <- A[]
expect_equal(res[, "a", drop=TRUE], c(22,26))

if (requireNamespace("bit64", quietly=TRUE)) {
suppressMessages(library(bit64))
uri <- tempfile()
dom <- tiledb_domain(dims = c(tiledb_dim("rows", c(as.integer64(1), as.integer64(4)), as.integer64(4), "INT64"),
tiledb_dim("cols", c(as.integer64(1), as.integer64(4)), as.integer64(4), "INT64"),
tiledb_dim("depth", c(as.integer64(1), as.integer64(4)), as.integer64(4), "INT64")))
schema <- tiledb_array_schema(dom, attrs = c(tiledb_attr("a", type = "INT64")))
tiledb_array_create(uri, schema)
data <- array(as.integer64(1:64), dim = c(4,4,4))
A <- tiledb_array(uri = uri)
A[] <- data

A <- tiledb_array(uri = uri, return_as="data.frame")
res <- A[2,2,2]
expect_equal(res[, "a", drop=TRUE], as.integer64(22))
res <- A[2,2:3,2]
expect_equal(res[, "a", drop=TRUE], as.integer64(c(22,26)))
selected_ranges(A) <- list(cbind(2,2), cbind(2,2), cbind(2,2))
res <- A[]
expect_equal(res[, "a", drop=TRUE], as.integer64(22))
selected_ranges(A) <- list(cbind(2,2), cbind(2,3), cbind(2,2))
res <- A[]
expect_equal(res[, "a", drop=TRUE], as.integer64(c(22,26)))
}