From dc8f711831ca7955ddecdcbb44c55ebe073245ab Mon Sep 17 00:00:00 2001 From: gsvic Date: Fri, 26 Mar 2021 12:45:28 +0200 Subject: [PATCH 1/2] switch getValiditBytemap to public --- src/main/java/io/tiledb/java/api/Query.java | 4 ++-- src/test/java/io/tiledb/java/api/QueryTest.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/io/tiledb/java/api/Query.java b/src/main/java/io/tiledb/java/api/Query.java index 06b31069..a89a2499 100644 --- a/src/main/java/io/tiledb/java/api/Query.java +++ b/src/main/java/io/tiledb/java/api/Query.java @@ -1573,9 +1573,9 @@ public long[] getOffsetArray(String bufferName) throws TileDBError { } /** @param attribute */ - Object getValidityByteMap(String attribute) throws TileDBError { + public short[] getValidityByteMap(String attribute) throws TileDBError { if (this.validityByteMaps_.containsKey(attribute)) { - return this.validityByteMaps_.get(attribute).toJavaArray(); + return (short[])this.validityByteMaps_.get(attribute).toJavaArray(); } throw new TileDBError("Attribute " + attribute + " is not nullable"); diff --git a/src/test/java/io/tiledb/java/api/QueryTest.java b/src/test/java/io/tiledb/java/api/QueryTest.java index f2df3a2a..1e49a459 100644 --- a/src/test/java/io/tiledb/java/api/QueryTest.java +++ b/src/test/java/io/tiledb/java/api/QueryTest.java @@ -1143,8 +1143,8 @@ public void denseArrayReadTest() throws Exception { byte[] a1 = (byte[]) query.getBuffer("a1"); float[] a2 = (float[]) query.getBuffer("a2"); - short[] a1ValidityByteMap = (short[]) query.getValidityByteMap("a1"); - short[] a2ValidityByteMap = (short[]) query.getValidityByteMap("a2"); + short[] a1ValidityByteMap = query.getValidityByteMap("a1"); + short[] a2ValidityByteMap = query.getValidityByteMap("a2"); Assert.assertArrayEquals(new int[] {1, 1, 2, 2}, dim1); Assert.assertArrayEquals(new int[] {1, 2, 1, 2}, dim2); From 3eeb5bc577f1b0e5b2ba5e3aff6b7e15730cf69e Mon Sep 17 00:00:00 2001 From: gsvic Date: Tue, 30 Mar 2021 19:40:48 +0300 Subject: [PATCH 2/2] set the correct validity bytemap size --- src/main/java/io/tiledb/java/api/Query.java | 32 +++++++++---------- .../java/io/tiledb/java/api/QueryTest.java | 24 +++++++------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/src/main/java/io/tiledb/java/api/Query.java b/src/main/java/io/tiledb/java/api/Query.java index a89a2499..076dd7d4 100644 --- a/src/main/java/io/tiledb/java/api/Query.java +++ b/src/main/java/io/tiledb/java/api/Query.java @@ -63,6 +63,7 @@ public class Query implements AutoCloseable { private Map> buffer_sizes_; private Map validityByteMaps_; private Map validityByteMapsByteBuffers_; + private Map validityByteMapSizes_; public Query(Array array, QueryType type) throws TileDBError { Context _ctx = array.getCtx(); @@ -85,6 +86,7 @@ public Query(Array array, QueryType type) throws TileDBError { this.buffer_sizes_ = Collections.synchronizedMap(new HashMap<>()); this.validityByteMaps_ = Collections.synchronizedMap(new HashMap<>()); this.validityByteMapsByteBuffers_ = Collections.synchronizedMap(new HashMap<>()); + this.validityByteMapSizes_ = Collections.synchronizedMap(new HashMap<>()); } public Query(Array array) throws TileDBError { @@ -538,23 +540,11 @@ public synchronized Query setBuffer(String attr, NativeArray buffer, long buffer * * @param attr The attribute name. * @param buffer NativeBuffer to be used for the attribute values. - * @param bufferElements The actual number of buffer elements * @param bytemap The byte-map * @exception TileDBError A TileDB exception */ - public synchronized Query setBufferNullable( - String attr, NativeArray buffer, long bufferElements, NativeArray bytemap) + public synchronized Query setBufferNullable(String attr, NativeArray buffer, NativeArray bytemap) throws TileDBError { - if (bufferElements <= 0) { - throw new TileDBError("Number of buffer elements must be >= 1"); - } - if (bufferElements > buffer.getSize()) { - throw new TileDBError( - "Number of elements requested exceeds the number of elements in allocated buffer: " - + bufferElements - + " > " - + buffer.getSize()); - } try (ArraySchema schema = array.getSchema()) { try (Domain domain = schema.getDomain()) { @@ -573,10 +563,10 @@ public synchronized Query setBufferNullable( uint64_tArray offsets_array_size = new uint64_tArray(1); uint64_tArray values_array_size = new uint64_tArray(1); uint64_tArray buffer_validity_bytemap_size = new uint64_tArray(1); + offsets_array_size.setitem(0, BigInteger.valueOf(0l)); - values_array_size.setitem(0, BigInteger.valueOf(bufferElements * buffer.getNativeTypeSize())); - buffer_validity_bytemap_size.setitem( - 0, BigInteger.valueOf(bytemap.getSize() * bytemap.getNativeTypeSize())); + values_array_size.setitem(0, BigInteger.valueOf(buffer.getNBytes())); + buffer_validity_bytemap_size.setitem(0, BigInteger.valueOf(bytemap.getNBytes())); Pair buffer_sizes = new Pair<>(offsets_array_size, values_array_size); @@ -589,6 +579,7 @@ public synchronized Query setBufferNullable( buffers_.put(attr, new Pair(null, buffer)); buffer_sizes_.put(attr, buffer_sizes); validityByteMaps_.put(attr, bytemap); + validityByteMapSizes_.put(attr, buffer_validity_bytemap_size); // Set the actual TileDB buffer uint64_tArray buffer_size = buffer_sizes.getSecond(); @@ -728,6 +719,7 @@ public synchronized Query setBufferNullable( buffers_.put(attr, new Pair<>(offsets, buffer)); buffer_sizes_.put(attr, buffer_sizes); validityByteMaps_.put(attr, bytemap); + validityByteMapSizes_.put(attr, buffer_validity_bytemap_size); ctx.handleError( tiledb.tiledb_query_set_buffer_var_nullable( @@ -1575,7 +1567,13 @@ public long[] getOffsetArray(String bufferName) throws TileDBError { /** @param attribute */ public short[] getValidityByteMap(String attribute) throws TileDBError { if (this.validityByteMaps_.containsKey(attribute)) { - return (short[])this.validityByteMaps_.get(attribute).toJavaArray(); + int nelements = + this.validityByteMapSizes_ + .get(attribute) + .getitem(0) + .divide(BigInteger.valueOf(Datatype.TILEDB_UINT8.getNativeSize())) + .intValue(); + return (short[]) this.validityByteMaps_.get(attribute).toJavaArray(nelements); } throw new TileDBError("Attribute " + attribute + " is not nullable"); diff --git a/src/test/java/io/tiledb/java/api/QueryTest.java b/src/test/java/io/tiledb/java/api/QueryTest.java index 1e49a459..fd02c114 100644 --- a/src/test/java/io/tiledb/java/api/QueryTest.java +++ b/src/test/java/io/tiledb/java/api/QueryTest.java @@ -1014,8 +1014,8 @@ public void denseArrayWrite() throws Exception { NativeArray a2Bytemap = new NativeArray(ctx, new short[] {1, 1, 1, 0}, Datatype.TILEDB_UINT8); - query.setBufferNullable("a1", a1, 4, a1Bytemap); - query.setBufferNullable("a2", a2, 4, a2Bytemap); + query.setBufferNullable("a1", a1, a1Bytemap); + query.setBufferNullable("a2", a2, a2Bytemap); // Submit query query.submit(); @@ -1074,7 +1074,7 @@ public void sparseArrayWrite() throws TileDBError { new NativeArray(ctx, new short[] {1, 1, 1, 0, 0}, Datatype.TILEDB_UINT8); query.setBuffer("d1", d_off, d_data); - query.setBufferNullable("a1", a1, 5, a1ByteMap); + query.setBufferNullable("a1", a1, a1ByteMap); query.setBufferNullable("a2", a2_off, a2_data, a2ByteMap); // Submit query @@ -1112,17 +1112,17 @@ public void denseArrayReadTest() throws Exception { query.addRange(1, 1, 2); query.setLayout(TILEDB_ROW_MAJOR); - NativeArray dim1Array = new NativeArray(ctx, 4, Integer.class); - NativeArray dim2Array = new NativeArray(ctx, 4, Integer.class); - NativeArray a1Array = new NativeArray(ctx, 4, Character.class); - NativeArray a1byteMap = new NativeArray(ctx, 4, Datatype.TILEDB_UINT8); - NativeArray a2Array = new NativeArray(ctx, 4, Float.class); - NativeArray a2byteMap = new NativeArray(ctx, 4, Datatype.TILEDB_UINT8); + NativeArray dim1Array = new NativeArray(ctx, 100, Integer.class); + NativeArray dim2Array = new NativeArray(ctx, 100, Integer.class); + NativeArray a1Array = new NativeArray(ctx, 100, Character.class); + NativeArray a1byteMap = new NativeArray(ctx, 100, Datatype.TILEDB_UINT8); + NativeArray a2Array = new NativeArray(ctx, 100, Float.class); + NativeArray a2byteMap = new NativeArray(ctx, 100, Datatype.TILEDB_UINT8); query.setBuffer("rows", dim1Array); query.setBuffer("cols", dim2Array); - query.setBufferNullable("a1", a1Array, 4, a1byteMap); - query.setBufferNullable("a2", a2Array, 4, a2byteMap); + query.setBufferNullable("a1", a1Array, a1byteMap); + query.setBufferNullable("a2", a2Array, a2byteMap); // Submit query query.submit(); @@ -1249,7 +1249,7 @@ public void sparseArrayReadTest() throws Exception { NativeArray a2Offsets = new NativeArray(ctx, 5, Datatype.TILEDB_UINT64); NativeArray a2byteMap = new NativeArray(ctx, 10, Datatype.TILEDB_UINT8); - query.setBufferNullable("a1", a1Array, 5, a1byteMap); + query.setBufferNullable("a1", a1Array, a1byteMap); query.setBufferNullable("a2", a2Offsets, a2Array, a2byteMap); // Submit query