diff --git a/src/main/java/io/tiledb/java/api/Query.java b/src/main/java/io/tiledb/java/api/Query.java index 9e53b91f..f2d7d662 100644 --- a/src/main/java/io/tiledb/java/api/Query.java +++ b/src/main/java/io/tiledb/java/api/Query.java @@ -28,6 +28,7 @@ import io.tiledb.libtiledb.*; import java.math.BigInteger; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -56,9 +57,9 @@ public class Query implements AutoCloseable { private NativeArray subarray; - private HashMap buffers_; - private HashMap> var_buffers_; - private HashMap> buffer_sizes_; + private Map buffers_; + private Map> var_buffers_; + private Map> buffer_sizes_; public Query(Array array, QueryType type) throws TileDBError { Context _ctx = array.getCtx(); @@ -76,9 +77,9 @@ public Query(Array array, QueryType type) throws TileDBError { this.array = array; this.querypp = _querypp; this.queryp = tiledb.tiledb_query_tpp_value(_querypp); - this.buffers_ = new HashMap<>(); - this.var_buffers_ = new HashMap<>(); - this.buffer_sizes_ = new HashMap<>(); + this.buffers_ = Collections.synchronizedMap(new HashMap<>()); + this.var_buffers_ = Collections.synchronizedMap(new HashMap<>()); + this.buffer_sizes_ = Collections.synchronizedMap(new HashMap<>()); } public Query(Array array) throws TileDBError { @@ -91,7 +92,7 @@ public Query(Array array) throws TileDBError { * @param layout The layout order to be set. * @exception TileDBError A TileDB exception */ - public Query setLayout(Layout layout) throws TileDBError { + public synchronized Query setLayout(Layout layout) throws TileDBError { ctx.handleError(tiledb.tiledb_query_set_layout(ctx.getCtxp(), queryp, layout.toSwigEnum())); return this; } @@ -150,7 +151,7 @@ public void submitAsync(Callback callback) throws TileDBError { * @param subarray The targeted subarray. * @exception TileDBError A TileDB exception */ - public Query setSubarray(NativeArray subarray) throws TileDBError { + public synchronized Query setSubarray(NativeArray subarray) throws TileDBError { Types.typeCheck(subarray.getNativeType(), array.getSchema().getDomain().getType()); ctx.handleError( tiledb.tiledb_query_set_subarray(ctx.getCtxp(), queryp, subarray.toVoidPointer())); @@ -171,7 +172,7 @@ public Query setSubarray(NativeArray subarray) throws TileDBError { * @return This query * @throws TileDBError A TileDB exception */ - public Query addRange(int dimIdx, Object start, Object end) throws TileDBError { + public synchronized Query addRange(int dimIdx, Object start, Object end) throws TileDBError { Datatype dimType; try (ArraySchema schema = array.getSchema(); Domain domain = schema.getDomain()) { @@ -264,7 +265,7 @@ public Pair getRange(int dimIdx, long rangeIdx) throws TileDBErr * @param buffer NativeBuffer to be used for the attribute values. * @exception TileDBError A TileDB exception */ - public Query setBuffer(String attr, NativeArray buffer) throws TileDBError { + public synchronized Query setBuffer(String attr, NativeArray buffer) throws TileDBError { try (ArraySchema schema = array.getSchema()) { if (attr.equals(tiledb.tiledb_coords())) { try (Domain domain = schema.getDomain()) { @@ -297,7 +298,8 @@ public Query setBuffer(String attr, NativeArray buffer) throws TileDBError { * @param buffer Buffer vector with elements of the attribute type. * @exception TileDBError A TileDB exception */ - public Query setBuffer(String attr, NativeArray offsets, NativeArray buffer) throws TileDBError { + public synchronized Query setBuffer(String attr, NativeArray offsets, NativeArray buffer) + throws TileDBError { if (attr.equals(tiledb.tiledb_coords())) { throw new TileDBError("Cannot set coordinate buffer as variable sized."); @@ -313,7 +315,7 @@ public Query setBuffer(String attr, NativeArray offsets, NativeArray buffer) thr Types.typeCheck(attribute.getType(), buffer.getNativeType()); } Pair buffer_sizes = - new Pair(new uint64_tArray(1), new uint64_tArray(1)); + new Pair<>(new uint64_tArray(1), new uint64_tArray(1)); buffer_sizes.getFirst().setitem(0, BigInteger.valueOf(offsets.getNBytes())); buffer_sizes.getSecond().setitem(0, BigInteger.valueOf(buffer.getNBytes())); // close previous buffers if they exist for this attribute @@ -327,6 +329,115 @@ public Query setBuffer(String attr, NativeArray offsets, NativeArray buffer) thr return this; } + private Query setBufferSizeUnsafe(String attribute, long offsetSize, long bufferSize) { + buffer_sizes_.get(attribute).getFirst().setitem(0, BigInteger.valueOf(offsetSize)); + buffer_sizes_.get(attribute).getSecond().setitem(0, BigInteger.valueOf(bufferSize)); + return this; + } + + public synchronized Query setBufferByteSize(String attribute, Long offsetSize, Long bufferSize) + throws TileDBError { + if (!var_buffers_.containsKey(attribute)) { + throw new TileDBError("Query var attribute buffer does not exist: " + attribute); + } + if (offsetSize <= 0 || bufferSize <= 0) { + throw new TileDBError("Number of buffer bytes must be >= 1"); + } + Pair varBuffers = var_buffers_.get(attribute); + NativeArray offsetBuffer = varBuffers.getFirst(); + Long offsetNBytes = offsetBuffer.getNBytes(); + NativeArray buffer = varBuffers.getSecond(); + Long bufferNBytes = buffer.getNBytes(); + if (offsetSize > offsetNBytes) { + throw new TileDBError( + "Number of offset bytes requested exceeds the number bytes of in allocated offset buffer: " + + offsetNBytes + + " > " + + offsetSize); + } + if (bufferSize > bufferNBytes) { + throw new TileDBError( + "Number of buffer bytes requested exceeds the number of bytes in allocated buffer" + + bufferNBytes + + " > " + + bufferSize); + } + return setBufferSizeUnsafe(attribute, offsetSize, bufferSize); + } + + public synchronized Query setBufferByteSize(String attribute, Long bufferSize) + throws TileDBError { + if (!buffers_.containsKey(attribute)) { + throw new TileDBError("Query attrbute buffer does not exist: " + attribute); + } + if (bufferSize <= 0) { + throw new TileDBError("Number of buffer bytes must be >= 1"); + } + NativeArray buffer = buffers_.get(attribute); + Long bufferNBytes = buffer.getNBytes(); + if (bufferSize > bufferNBytes) { + throw new TileDBError( + "Number of bytes requested exceeds the number of bytes in allocated buffer: " + + bufferSize + + " > " + + bufferNBytes); + } + return setBufferSizeUnsafe(attribute, 0l, bufferSize); + } + + public synchronized Query setBufferElements(String attribute, Integer bufferElements) + throws TileDBError { + if (!buffers_.containsKey(attribute)) { + throw new TileDBError("Query attribute buffer does not exist: " + attribute); + } + if (bufferElements <= 0) { + throw new TileDBError("Number of buffer elements must be >= 1"); + } + NativeArray buffer = buffers_.get(attribute); + Integer bufferSize = buffer.getSize(); + if (bufferElements > bufferSize) { + throw new TileDBError( + "Number of elements requested exceeds the number of elements in allocated buffer: " + + bufferElements + + " > " + + bufferSize); + } + return setBufferSizeUnsafe(attribute, 0l, (long) (bufferElements * buffer.getNativeTypeSize())); + } + + public synchronized Query setBufferElements( + String attribute, Integer offsetElements, Integer bufferElements) throws TileDBError { + if (!var_buffers_.containsKey(attribute)) { + throw new TileDBError("Query var attribute buffer does not exist: " + attribute); + } + if (offsetElements <= 0 || bufferElements <= 0) { + throw new TileDBError("Number of buffer elements must be >= 1"); + } + Pair varBuffers = var_buffers_.get(attribute); + NativeArray offsetBuffer = varBuffers.getFirst(); + Integer offsetSize = offsetBuffer.getSize(); + NativeArray buffer = varBuffers.getSecond(); + Integer bufferSize = buffer.getSize(); + if (offsetElements > offsetSize) { + throw new TileDBError( + "Number of offset elements requested exceeds the number of elements in allocated offset buffer: " + + offsetElements + + " > " + + offsetSize); + } + if (bufferElements > bufferSize) { + throw new TileDBError( + "Number of buffer elements requested exceeds the number of elements in allocated buffer" + + bufferElements + + " > " + + bufferSize); + } + return setBufferSizeUnsafe( + attribute, + (long) (offsetElements * offsetBuffer.getNativeTypeSize()), + (long) (bufferElements * buffer.getNativeTypeSize())); + } + /** * Set the coordinate buffer * @@ -364,7 +475,7 @@ public HashMap> resultBufferElements() throws TileDBErr BigInteger val_nbytes = buffer_sizes_.get(name).getSecond().getitem(0); Long nelements = val_nbytes.divide(BigInteger.valueOf(val_buffer.getNativeTypeSize())).longValue(); - result.put(name, new Pair(0l, nelements)); + result.put(name, new Pair<>(0l, nelements)); } for (Map.Entry> entry : var_buffers_.entrySet()) { String name = entry.getKey(); @@ -385,7 +496,7 @@ public HashMap> resultBufferElements() throws TileDBErr } /** Clears all attribute buffers. */ - public void resetBuffers() { + public synchronized void resetBuffers() { for (NativeArray buffer : buffers_.values()) { buffer.close(); } @@ -402,23 +513,25 @@ public void resetBuffers() { buffer_sizes_.clear(); } - /** Resets all attribute buffer sizes to zero */ - public void resetBufferSizes() { - BigInteger zero = BigInteger.valueOf(0L); + public synchronized Query resetBufferSizes(Long val) { + BigInteger sizeVal = BigInteger.valueOf(val); for (Pair size_pair : buffer_sizes_.values()) { - size_pair.getFirst().setitem(0, zero); - size_pair.getSecond().setitem(0, zero); + size_pair.getFirst().setitem(0, sizeVal); + size_pair.getSecond().setitem(0, sizeVal); } + return this; } - private void prepareSubmission() throws TileDBError { + /** Resets all attribute buffer sizes to zero */ + public Query resetBufferSizes() { + return resetBufferSizes(0l); + } + + private synchronized void prepareSubmission() throws TileDBError { for (Map.Entry entry : buffers_.entrySet()) { String name = entry.getKey(); NativeArray buffer = entry.getValue(); Pair buffer_sizes = buffer_sizes_.get(name); - buffer_sizes.getFirst().setitem(0, BigInteger.valueOf(0L)); - buffer_sizes.getSecond().setitem(0, BigInteger.valueOf(buffer.getNBytes())); - uint64_tArray buffer_size = buffer_sizes.getSecond(); ctx.handleError( tiledb.tiledb_query_set_buffer( @@ -430,9 +543,6 @@ private void prepareSubmission() throws TileDBError { NativeArray val_buffer = entry.getValue().getSecond(); Pair buffer_size = buffer_sizes_.get(name); - buffer_size.getFirst().setitem(0, BigInteger.valueOf(off_buffer.getNBytes())); - buffer_size.getSecond().setitem(0, BigInteger.valueOf(val_buffer.getNBytes())); - uint64_tArray offsets = PointerUtils.uint64_tArrayFromVoid(off_buffer.toVoidPointer()); uint64_tArray off_size = buffer_size.getFirst(); uint64_tArray val_size = buffer_size.getSecond(); @@ -535,7 +645,7 @@ public String toString() { /** Free's native TileDB resources associated with the Query object */ @Override - public void close() { + public synchronized void close() { if (queryp != null) { for (Pair size_pair : buffer_sizes_.values()) { size_pair.getFirst().delete(); diff --git a/src/test/java/io/tiledb/java/api/QueryTest.java b/src/test/java/io/tiledb/java/api/QueryTest.java new file mode 100644 index 00000000..849d28fe --- /dev/null +++ b/src/test/java/io/tiledb/java/api/QueryTest.java @@ -0,0 +1,141 @@ +package io.tiledb.java.api; + +import static io.tiledb.java.api.ArrayType.TILEDB_DENSE; +import static io.tiledb.java.api.Layout.TILEDB_ROW_MAJOR; +import static io.tiledb.java.api.QueryType.TILEDB_READ; +import static io.tiledb.java.api.QueryType.TILEDB_WRITE; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashMap; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class QueryTest { + + private Context ctx; + private String arrayURI = "query"; + + @Before + public void setup() throws Exception { + ctx = new Context(); + if (Files.exists(Paths.get(arrayURI))) { + TileDBObject.remove(ctx, arrayURI); + } + arrayCreate(); + arrayWrite(); + } + + @After + public void teardown() throws Exception { + if (Files.exists(Paths.get(arrayURI))) { + TileDBObject.remove(ctx, arrayURI); + } + } + + @Test + public void test() throws Exception { + arrayRead(); + } + + public void arrayCreate() throws Exception { + // The array will be 4x4 with dimensions "rows" and "cols", with domain [1,4]. + Dimension rows = + new Dimension(ctx, "rows", Integer.class, new Pair(1, 4), 2); + Dimension cols = + new Dimension(ctx, "cols", Integer.class, new Pair(1, 4), 2); + + // Create and set getDomain + Domain domain = new Domain(ctx); + domain.addDimension(rows); + domain.addDimension(cols); + + // Add two attributes "a1" and "a2", so each (i,j) cell can store + // a character on "a1" and a vector of two floats on "a2". + Attribute a1 = new Attribute(ctx, "a1", Character.class); + Attribute a2 = new Attribute(ctx, "a2", Float.class); + a2.setCellValNum(2); + + ArraySchema schema = new ArraySchema(ctx, TILEDB_DENSE); + schema.setTileOrder(TILEDB_ROW_MAJOR); + schema.setCellOrder(TILEDB_ROW_MAJOR); + schema.setDomain(domain); + schema.addAttribute(a1); + schema.addAttribute(a2); + + Array.create(arrayURI, schema); + } + + public void arrayWrite() throws Exception { + // Prepare cell buffers + NativeArray a1 = new NativeArray(ctx, "abcdefghijklmnop", String.class); + NativeArray a2 = + new NativeArray( + ctx, + new float[] { + 0.1f, 0.2f, 1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f, + 4.1f, 4.2f, 5.1f, 5.2f, 6.1f, 6.2f, 7.1f, 7.2f, + 8.1f, 8.2f, 9.1f, 9.2f, 10.1f, 10.2f, 11.1f, 11.2f, + 12.1f, 12.2f, 13.1f, 13.2f, 14.1f, 14.2f, 15.1f, 15.2f + }, + Float.class); + + // Create query + Array array = new Array(ctx, arrayURI, TILEDB_WRITE); + Query query = new Query(array); + query.setLayout(TILEDB_ROW_MAJOR); + query.setBuffer("a1", a1); + query.setBuffer("a2", a2); + // Submit query + query.submit(); + query.close(); + array.close(); + } + + private void arrayRead() throws Exception { + Array array = new Array(ctx, arrayURI, TILEDB_READ); + + // Create query + Query query = new Query(array, TILEDB_READ); + + // Slice only rows 1, 2 and cols 2, 3, 4 + query.addRange(0, (int) 1, (int) 2); + query.addRange(1, (int) 2, (int) 4); + query.setLayout(TILEDB_ROW_MAJOR); + + // Prepare the vector that will hold the result + // (of size 6 elements for "a1" and 12 elements for "a2" since + // it stores two floats per cell) + + NativeArray a1Array = new NativeArray(ctx, 6, Character.class); + NativeArray a2Array = new NativeArray(ctx, 12, Float.class); + query.setBuffer("a1", a1Array); + query.setBuffer("a2", a2Array); + + // set the number of buffer elements to a size smaller than the read buffer size + query.setBufferElements("a1", 3); + query.setBufferElements("a2", 6); + + // Submit query + query.submit(); + + HashMap> resultElements = query.resultBufferElements(); + + Assert.assertEquals(Long.valueOf(3), resultElements.get("a1").getSecond()); + Assert.assertEquals(Long.valueOf(6), resultElements.get("a2").getSecond()); + + byte[] a1 = (byte[]) query.getBuffer("a1"); + float[] a2 = (float[]) query.getBuffer("a2"); + + Assert.assertArrayEquals(new byte[] {'b', 'c', 'd'}, a1); + float[] expected_a2 = new float[] {1.1f, 1.2f, 2.1f, 2.2f, 3.1f, 3.2f}; + Assert.assertArrayEquals(expected_a2, a2, 0.01f); + + a1Array.close(); + a2Array.close(); + query.close(); + array.close(); + } +}