Wrapping TileDB 2.5.0 methods

Author: DimitrisStaratzis

cmake/Modules/FindTileDB_EP.cmake

@@ -48,14 +48,14 @@ if (NOT TILEDB_FOUND)
     # Try to download prebuilt artifacts unless the user specifies to build from source
     if(DOWNLOAD_TILEDB_PREBUILT)
         if (WIN32) # Windows
-          SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.5.0/tiledb-windows-x86_64-2.5.0-d11292d.zip")
-          SET(DOWNLOAD_SHA1 "9db08d120fbaab2d635b086c1d2861a824994461")
-        elseif(APPLE) # OSX
-          SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.5.0/tiledb-macos-x86_64-2.5.0-d11292d.tar.gz")
-          SET(DOWNLOAD_SHA1 "daa6791eb08c876b127292cfe084f9de7a799766")
+          SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.5.1/tiledb-windows-x86_64-2.5.1-5b65a96.zip")
+          SET(DOWNLOAD_SHA1 "d08658d08fab8da1c6d21c6c7af68b02d9d5d9e7")
+        elseif(APPLE) # macOS
+          SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.5.1/tiledb-macos-x86_64-2.5.1-5b65a96.tar.gz")
+          SET(DOWNLOAD_SHA1 "66521ccf7b5e6fe9be29c0107e8bb433bb50cfad")
         else() # Linux
-          SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.5.0/tiledb-linux-x86_64-2.5.0-d11292d.tar.gz")
-          SET(DOWNLOAD_SHA1 "937b08fc58f0783d5738175dfebc23f69a512ca1")
+          SET(DOWNLOAD_URL "https://github.com/TileDB-Inc/TileDB/releases/download/2.5.1/tiledb-linux-x86_64-2.5.1-5b65a96.tar.gz")
+          SET(DOWNLOAD_SHA1 "d276396e0242c64d54479018517ad93206dc9e1b")
         endif()
 
         ExternalProject_Add(ep_tiledb

gradle.properties

@@ -1,5 +1,5 @@
 TILEDB_GIT_REPOSITORY=https://github.com/TileDB-Inc/TileDB
-TILEDB_GIT_TAG=2.5.0
+TILEDB_GIT_TAG=2.5.1
 TILEDB_VERBOSE=ON
 TILEDB_S3=ON
 TILEDB_AZURE=OFF

src/main/java/io/tiledb/java/api/Array.java

@@ -411,6 +411,23 @@ public void evolve(Context ctx, ArraySchemaEvolution schemaEvolution) throws Til
     }
   }
 
+  /**
+   * Upgrades an array to the latest format version.
+   *
+   * <p>**Example:** String uri = "test_array" array.upgradeVersion(ctx)
+   *
+   * @param ctx The TileDB context.
+   * @param config Configuration parameters for the upgrade (`nullptr` means default, which will use
+   *     the config from `ctx`).
+   */
+  public void upgradeVersion(Context ctx, Config config) throws TileDBError {
+    try {
+      ctx.handleError(tiledb.tiledb_array_upgrade_version(ctx.getCtxp(), uri, config.getConfigp()));
+    } catch (TileDBError err) {
+      throw err;
+    }
+  }
+
   /**
    * Cleans up the array, such as consolidated fragments and array metadata. Note that this will
    * coarsen the granularity of time traveling (see docs for more information).

src/main/java/io/tiledb/java/api/FragmentInfo.java

@@ -419,4 +419,191 @@ public String dump() throws TileDBError {
 
     return tiledb.charpp_value(uri);
   }
+
+  /**
+   * Retrieves the number of MBRs from the fragment.
+   *
+   * <p>In the case of sparse fragments, this is the number of physical tiles.
+   *
+   * <p>Dense fragments do not contain MBRs.
+   *
+   * @param fragmentID The index of the fragment of interest.
+   * @return The number of MBRs.
+   * @throws TileDBError
+   */
+  public long getMBRNum(long fragmentID) throws TileDBError {
+    SWIGTYPE_p_unsigned_long_long numFrags = tiledb.new_ullp();
+    ctx.handleError(
+        tiledb.tiledb_fragment_info_get_mbr_num(
+            ctx.getCtxp(), this.fragmentInfop, fragmentID, numFrags));
+    return tiledb.ullp_value(numFrags).longValue();
+  }
+
+  /**
+   * Retrieves the MBR from a given fragment for a given dimension index.
+   *
+   * @param fragmentID The index of the fragment of interest.
+   * @param mid The mbr of the fragment of interest.
+   * @param dimensionID The dimension index, following the order as it was defined in the domain of
+   *     the array schema.
+   * @return The MBR.
+   */
+  public long[] getMBRFromIndex(long fragmentID, long dimensionID, long mid) throws TileDBError {
+    long[] mbr;
+    try (NativeArray mbrArray = new NativeArray(ctx, 2, Long.class)) {
+      ctx.handleError(
+          tiledb.tiledb_fragment_info_get_mbr_from_index(
+              ctx.getCtxp(),
+              this.fragmentInfop,
+              fragmentID,
+              mid,
+              dimensionID,
+              mbrArray.toVoidPointer()));
+      mbr = (long[]) mbrArray.toJavaArray();
+    }
+    return mbr;
+  }
+
+  /**
+   * Retrieves the MBR from a given fragment for a given dimension index.
+   *
+   * @param fragmentID The index of the fragment of interest.
+   * @param mid The mbr of the fragment of interest.
+   * @param dimName The dimension name.
+   * @return The MBR.
+   */
+  public long[] getMBRFromName(long fragmentID, String dimName, long mid) throws TileDBError {
+    long[] mbr;
+    try (NativeArray mbrArray = new NativeArray(ctx, 2, Long.class)) {
+      ctx.handleError(
+          tiledb.tiledb_fragment_info_get_mbr_from_name(
+              ctx.getCtxp(),
+              this.fragmentInfop,
+              fragmentID,
+              mid,
+              dimName,
+              mbrArray.toVoidPointer()));
+      mbr = (long[]) mbrArray.toJavaArray();
+    }
+    return mbr;
+  }
+
+  /**
+   * Returns the MBR of the fragment with the given index on the given dimension index. Applicable
+   * to string dimensions.
+   *
+   * @param fragmentID The index of the fragment of interest.
+   * @param mid The mbr of the fragment of interest.
+   * @param dimId The dimension index, following the order as it was defined in the domain of the
+   *     array schema.
+   * @return The MBR.
+   */
+  public Pair<Long, Long> getMBRVarSizeFromIndex(long fragmentID, long dimId, long mid)
+      throws TileDBError {
+    SWIGTYPE_p_unsigned_long_long startSize = tiledb.new_ullp();
+    SWIGTYPE_p_unsigned_long_long endSize = tiledb.new_ullp();
+    ctx.handleError(
+        tiledb.tiledb_fragment_info_get_mbr_var_size_from_index(
+            ctx.getCtxp(), this.fragmentInfop, fragmentID, mid, dimId, startSize, endSize));
+    return new Pair<>(
+        tiledb.ullp_value(startSize).longValue(), tiledb.ullp_value(endSize).longValue());
+  }
+
+  /**
+   * Returns the MBR of the fragment with the given index on the given dimension name. Applicable to
+   * string dimensions.
+   *
+   * @param fragmentID The index of the fragment of interest.
+   * @param mid The mbr of the fragment of interest.
+   * @param dimName The dimension name.
+   * @return The MBR.
+   */
+  public Pair<Long, Long> getMBRVarSizeFromName(long fragmentID, String dimName, long mid)
+      throws TileDBError {
+    SWIGTYPE_p_unsigned_long_long startSize = tiledb.new_ullp();
+    SWIGTYPE_p_unsigned_long_long endSize = tiledb.new_ullp();
+    ctx.handleError(
+        tiledb.tiledb_fragment_info_get_mbr_var_size_from_name(
+            ctx.getCtxp(), this.fragmentInfop, fragmentID, mid, dimName, startSize, endSize));
+    return new Pair<>(
+        tiledb.ullp_value(startSize).longValue(), tiledb.ullp_value(endSize).longValue());
+  }
+
+  /**
+   * Returns the MBR of the fragment with the given index on the given dimension index. Applicable
+   * to string dimensions.
+   *
+   * @param fragmentID The index of the fragment of interest.
+   * @param mid The mbr of the fragment of interest.
+   * @param dimId The dimension index, following the order as it was defined in the domain of the
+   *     array schema.
+   * @return The MBR.
+   */
+  public Pair<String, String> getMBRVarFromIndex(long fragmentID, long dimId, long mid)
+      throws TileDBError {
+    Pair<Long, Long> size = this.getMBRVarSizeFromIndex(fragmentID, dimId, mid);
+
+    try (NativeArray start = new NativeArray(ctx, size.getFirst().intValue(), String.class);
+        NativeArray end = new NativeArray(ctx, size.getSecond().intValue(), String.class); ) {
+      ctx.handleError(
+          tiledb.tiledb_fragment_info_get_mbr_var_from_index(
+              ctx.getCtxp(),
+              this.fragmentInfop,
+              fragmentID,
+              mid,
+              dimId,
+              start.toVoidPointer(),
+              end.toVoidPointer()));
+      Object st = new String((byte[]) start.toJavaArray());
+      Object e = new String((byte[]) end.toJavaArray());
+      return new Pair(st, e);
+    }
+  }
+
+  /**
+   * Returns the MBR of the fragment with the given index on the given dimension name. Applicable to
+   * string dimensions.
+   *
+   * @param fragmentID The index of the fragment of interest.
+   * @param mid The mbr of the fragment of interest.
+   * @param dimName The dimension name.
+   * @return The MBR.
+   */
+  public Pair<String, String> getMBRVarFromName(long fragmentID, String dimName, long mid)
+      throws TileDBError {
+    Pair<Long, Long> size = this.getMBRVarSizeFromName(fragmentID, dimName, mid);
+
+    try (NativeArray start = new NativeArray(ctx, size.getFirst().intValue(), String.class);
+        NativeArray end = new NativeArray(ctx, size.getSecond().intValue(), String.class); ) {
+      ctx.handleError(
+          tiledb.tiledb_fragment_info_get_mbr_var_from_name(
+              ctx.getCtxp(),
+              this.fragmentInfop,
+              fragmentID,
+              mid,
+              dimName,
+              start.toVoidPointer(),
+              end.toVoidPointer()));
+      Object st = new String((byte[]) start.toJavaArray());
+      Object e = new String((byte[]) end.toJavaArray());
+      return new Pair(st, e);
+    }
+  }
+
+  /**
+   * Get the fragment info schema name.
+   *
+   * @param fragmentID The fragment info object.
+   * @return The schema name.
+   * @throws TileDBError
+   */
+  public String getArraySchemaName(long fragmentID) throws TileDBError {
+    SWIGTYPE_p_p_char name = tiledb.new_charpp();
+
+    ctx.handleError(
+        tiledb.tiledb_fragment_info_get_array_schema_name(
+            ctx.getCtxp(), fragmentInfop, fragmentID, name));
+
+    return tiledb.charpp_value(name);
+  }
 }

src/test/java/io/tiledb/java/api/ArrayTest.java

@@ -6,9 +6,14 @@
 import static io.tiledb.java.api.QueryType.TILEDB_READ;
 import static io.tiledb.java.api.QueryType.TILEDB_WRITE;
 
+import java.io.File;
+import java.io.IOException;
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Map;
+import org.apache.commons.io.FileUtils;
 import org.junit.*;
 import org.junit.rules.TemporaryFolder;
 
@@ -37,6 +42,11 @@ public void tearDown() throws Exception {
     ctx.close();
   }
 
+  private String testArrayURIString(String arrayName) {
+    Path arraysPath = Paths.get("src", "test", "resources", "data", arrayName);
+    return "file://".concat(arraysPath.toAbsolutePath().toString());
+  }
+
   private Object[] getArray(Object val) {
     if (val instanceof Object[]) return (Object[]) val;
     int arrlength = java.lang.reflect.Array.getLength(val);
@@ -761,4 +771,73 @@ public void testArrayReopen() throws Exception {
       array.reopen();
     }
   }
+
+  @Test
+  public void testArrayUpgrade() throws TileDBError, IOException {
+
+    // move old array to new temp folder for the upgrade.
+    String source = "src/test/resources/data/1.6/quickstart_sparse_array";
+    File srcDir = new File(source);
+
+    String destination = "src/test/resources/data/quickstart_sparse_array_v2";
+    File destDir = new File(destination);
+
+    try {
+      FileUtils.copyDirectory(srcDir, destDir);
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+
+    // READ BEFORE UPGRADE
+    Array array = new Array(ctx, testArrayURIString("quickstart_sparse_array_v2"), TILEDB_READ);
+    NativeArray subarray = new NativeArray(ctx, new int[] {1, 4, 1, 4}, Integer.class);
+
+    // Create query
+    Query query = new Query(array, TILEDB_READ);
+    query.setLayout(TILEDB_ROW_MAJOR);
+    query.setSubarray(subarray);
+    query.setBuffer("a", new NativeArray(ctx, 16, Integer.class));
+    query.setBuffer("rows", new NativeArray(ctx, 16, Integer.class));
+    query.setBuffer("cols", new NativeArray(ctx, 16, Integer.class));
+
+    // Submit query
+    query.submit();
+
+    int[] rows = (int[]) query.getBuffer("rows");
+    int[] cols = (int[]) query.getBuffer("cols");
+    int[] data = (int[]) query.getBuffer("a");
+    query.close();
+
+    // upgrade array
+    array.upgradeVersion(ctx, array.getConfig());
+
+    // check schema
+    array.getSchema().check();
+
+    // READ AFTER UPGRADE
+    query = new Query(array, TILEDB_READ);
+    query.setLayout(TILEDB_ROW_MAJOR);
+    query.setSubarray(subarray);
+    query.setBuffer("a", new NativeArray(ctx, 16, Integer.class));
+    query.setBuffer("rows", new NativeArray(ctx, 16, Integer.class));
+    query.setBuffer("cols", new NativeArray(ctx, 16, Integer.class));
+
+    // Submit query
+    query.submit();
+
+    int[] rowsAfter = (int[]) query.getBuffer("rows");
+    int[] colsAfter = (int[]) query.getBuffer("cols");
+    int[] dataAfter = (int[]) query.getBuffer("a");
+
+    // clean up
+    query.close();
+    array.close();
+
+    // compare
+    Assert.assertArrayEquals(cols, colsAfter);
+    Assert.assertArrayEquals(rows, rowsAfter);
+    Assert.assertArrayEquals(data, dataAfter);
+
+    FileUtils.deleteDirectory(destDir);
+  }
 }