diff --git a/.github/workflows/github_actions.yml b/.github/workflows/github_actions.yml index 1c6ef01..436b658 100644 --- a/.github/workflows/github_actions.yml +++ b/.github/workflows/github_actions.yml @@ -1,13 +1,46 @@ name: TileDB-Cloud-Java on: + workflow_dispatch: push: tags: - '*' + branches: + - master + - test-* + pull_request: jobs: + Examples: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v3 + + - run: | + unset SYSTEM + ./gradlew assemble; + shell: bash + name: assemble + + - name: Run examples + env: + API_TOKEN: ${{ secrets.API_TOKEN }} + run: | + cd build/libs + rm *javadoc* *sources* *default* + p=$(readlink -f *.jar) + cd ../../src/main/java/examples + javac -cp $p *.java + for x in $(ls -1 *.java | sed 's/.java//'); + do + echo $x; + class_name=examples.$x; + java -cp $p $class_name; + done Release: if: startsWith(github.ref, 'refs/tags/') + needs: [Examples] name: Create-Release runs-on: ubuntu-latest steps: diff --git a/build.gradle b/build.gradle index a119f57..f8ed96f 100644 --- a/build.gradle +++ b/build.gradle @@ -27,7 +27,7 @@ apply plugin: 'java' apply plugin: 'com.diffplug.spotless' group 'io.tiledb' -version = '0.1.1-SNAPSHOT' +version = '0.2.1-SNAPSHOT' repositories { mavenCentral() @@ -158,12 +158,10 @@ dependencies { implementation group: 'org.apache.arrow', name: 'arrow-vector', version: '9.0.0' implementation group: 'org.apache.arrow', name: 'arrow-compression', version: '9.0.0' implementation group: 'org.apache.arrow', name: 'arrow-memory-unsafe', version: '9.0.0' - implementation 'io.tiledb:tiledb-java:0.17.2' testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.2' testImplementation 'org.mockito:mockito-core:3.12.4' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.2' testImplementation 'junit:junit:4.13.1' - } import org.gradle.plugins.signing.Sign diff --git a/src/main/java/examples/Examples.java b/src/main/java/examples/Examples.java index c6ae13f..0e211a0 100644 --- a/src/main/java/examples/Examples.java +++ b/src/main/java/examples/Examples.java @@ -1,14 +1,13 @@ package examples; // Import classes: -import io.tiledb.cloud.TileDBClient; -import io.tiledb.cloud.TileDBUDF; -import io.tiledb.cloud.rest_api.ApiClient; +import io.tiledb.cloud.*; import io.tiledb.cloud.rest_api.ApiException; -import io.tiledb.cloud.TileDBLogin; import io.tiledb.cloud.rest_api.api.GroupsApi; import io.tiledb.cloud.rest_api.api.ArrayApi; import io.tiledb.cloud.rest_api.model.*; +import jdk.swing.interop.SwingInterOpUtils; +import org.apache.arrow.vector.ValueVector; import java.math.BigDecimal; import java.util.ArrayList; @@ -16,6 +15,8 @@ import java.util.HashMap; import java.util.List; +import static io.tiledb.cloud.TileDBUtils.serializeArgs; + public class Examples { public static void main(String[] args) { @@ -24,7 +25,7 @@ public static void main(String[] args) { TileDBClient tileDBClient = new TileDBClient( new TileDBLogin(null, null, - "", + System.getenv("API_TOKEN"), true, true, true)); @@ -33,19 +34,35 @@ public static void main(String[] args) { // to pass any credentials from now on. Just create the client as follows: // TileDBClient tileDBClient = new TileDBClient(); - ArrayApi apiInstance = new ArrayApi(tileDBClient.getApiClient()); // Uncomment to run whichever example you want -// runGenericUDF(tileDBClient); -// runArrayUDF(tileDBClient); -// runMultiArrayUDF(tileDBClient); -// getArraySchema(apiInstance); -// createArray(apiInstance); -// registerArray(apiInstance); -// listArrays(apiInstance); -// listGroups(defaultClient); -// deleteArray(apiInstance); -// deregisterArray(apiInstance); + runGenericUDF(tileDBClient); + runArrayUDF(tileDBClient); + runMultiArrayUDF(tileDBClient); + getArraySchema(tileDBClient); +// createArray(tileDBClient); +// registerArray(tileDBClient); + listArrays(tileDBClient); + listGroups(tileDBClient); +// deleteArray(tileDBClient); +// deregisterArray(tileDBClient); + runSQL(tileDBClient); + runSQLArrow(tileDBClient); + + } + + private static void runSQL(TileDBClient tileDBClient) { + SQLParameters sqlParameters = new SQLParameters(); + sqlParameters.setQuery("SELECT * FROM `tiledb://TileDB-Inc/quickstart_sparse`"); + TileDBSQL tileDBSQL = new TileDBSQL(tileDBClient, "TileDB-Inc", sqlParameters); + System.out.println(tileDBSQL.exec()); + } + + private static void runSQLArrow(TileDBClient tileDBClient) { + SQLParameters sqlParameters = new SQLParameters(); + sqlParameters.setQuery("SELECT * FROM `tiledb://TileDB-Inc/quickstart_sparse`"); + TileDBSQL tileDBSQL = new TileDBSQL(tileDBClient, "TileDB-Inc", sqlParameters); + Pair, Integer> a = tileDBSQL.execArrow(); } /** @@ -59,11 +76,12 @@ private static void runGenericUDF(TileDBClient tileDBClient){ HashMap arguments = new HashMap<>(); arguments.put("arg1", "a1"); arguments.put("arg2", "a2"); - System.out.println(tileDBUDF.executeGeneric(genericUDF, arguments)); //could be JSON or Arrow + genericUDF.setArgument(serializeArgs(arguments)); + System.out.println(tileDBUDF.executeGeneric(genericUDF)); //could be JSON or Arrow } /** - * Runs an array UDF on a TileDB Arrray + * Runs an array UDF on a TileDB Array * @param tileDBClient */ public static void runArrayUDF(TileDBClient tileDBClient){ @@ -80,15 +98,18 @@ public static void runArrayUDF(TileDBClient tileDBClient){ queryRanges.addRangesItem(range1); queryRanges.addRangesItem(range2); - HashMap argumentsForArrayUDF = new HashMap<>(); argumentsForArrayUDF.put("attr", "rows"); argumentsForArrayUDF.put("scale", 9); - MultiArrayUDF multiArrayUDF = new MultiArrayUDF(); - multiArrayUDF.setUdfInfoName("TileDB-Inc/array-udf"); - multiArrayUDF.setRanges(queryRanges); - System.out.println(tileDBUDF.executeSingleArray(multiArrayUDF, argumentsForArrayUDF, "tiledb://TileDB-Inc/quickstart_sparse", "TileDB-Inc")); + GenericUDF genericUDF = new GenericUDF(); + genericUDF.setUdfInfoName("TileDB-Inc/array-udf"); + genericUDF.setArgument(serializeArgs(argumentsForArrayUDF)); + + UDFArrayDetails array = new UDFArrayDetails(); + array.setUri("tiledb://TileDB-Inc/quickstart_sparse"); + + System.out.println(tileDBUDF.executeSingleArray(genericUDF, array, queryRanges, "TileDB-Inc")); } /** @@ -117,7 +138,7 @@ public static void runMultiArrayUDF(TileDBClient tileDBClient){ //array1 UDFArrayDetails array1 = new UDFArrayDetails(); - array1.setUri("tiledb://TileDB-Inc/dense-array"); + array1.setUri("tiledb://shaunreed/dense-array"); array1.setRanges(queryRanges); array1.setBuffers(Arrays.asList("rows", "cols", "a1")); arrays.add(array1); @@ -135,17 +156,19 @@ public static void runMultiArrayUDF(TileDBClient tileDBClient){ arguments.put("attr1", "a1"); arguments.put("attr2", "a"); - System.out.println(tileDBUDF.executeMultiArray(multiArrayUDF, arguments)); + multiArrayUDF.setArgument(serializeArgs(arguments)); + + System.out.println(tileDBUDF.executeMultiArray(multiArrayUDF)); } /** * Deregister an array - * @param apiInstance */ - private static void deregisterArray(ArrayApi apiInstance) + private static void deregisterArray(TileDBClient tileDBClient) { String namespace = ""; // String | namespace array is in (an organization name or user's username) String array = ""; // String | name/uri of array that is url-encoded + ArrayApi apiInstance = new ArrayApi(tileDBClient.getApiClient()); try { apiInstance.deregisterArray(namespace, array); } catch (ApiException e) { @@ -159,13 +182,13 @@ private static void deregisterArray(ArrayApi apiInstance) /** * Delete an array - * @param apiInstance */ - private static void deleteArray(ArrayApi apiInstance) + private static void deleteArray(TileDBClient tileDBClient) { String namespace = ""; // String | namespace array is in (an organization name or user's username) String array = ""; // String | name/uri of array that is url-encoded String contentType = "application/json"; // String | Content Type of input and return mime + ArrayApi apiInstance = new ArrayApi(tileDBClient.getApiClient()); try { apiInstance.deleteArray(namespace, array, contentType); } catch (ApiException e) { @@ -177,17 +200,13 @@ private static void deleteArray(ArrayApi apiInstance) } } - /** - * List groups - * @param defaultClient - */ - private static void listGroups(ApiClient defaultClient) + private static void listGroups(TileDBClient tileDBClient) { - GroupsApi apiInstance = new GroupsApi(defaultClient); + GroupsApi apiInstance = new GroupsApi(tileDBClient.getApiClient()); Integer page = null; // Integer | pagination offset Integer perPage = null; // Integer | pagination limit String search = null; // String | search string that will look at name, namespace or description fields - String namespace = ""; // String | namespace + String namespace = "TileDB-Inc"; // String | namespace String orderby = null; // String | sort by which field valid values include last_accessed, size, name String permissions = null; // String | permissions valid values include read, read_write, write, admin List tag = null; // List | tag to search for, more than one can be included @@ -208,13 +227,11 @@ private static void listGroups(ApiClient defaultClient) } - /** - * List arrays - * @param apiInstance - */ - private static void listArrays(ArrayApi apiInstance) + private static void listArrays(TileDBClient tileDBClient) { - String namespace = ""; // String | namespace array is in (an organization name or user's username) + String namespace = "dstara"; // String | namespace array is in (an organization name or user's username) + ArrayApi apiInstance = new ArrayApi(tileDBClient.getApiClient()); + try { List result = apiInstance.getArraysInNamespace(namespace); System.out.println(result); @@ -228,12 +245,14 @@ private static void listArrays(ArrayApi apiInstance) } - private static void getArraySchema(ArrayApi arrayApi){ - String namespace = ""; // String | namespace array is in (an organization name or user's username) - String array = ""; // String | name/uri of array that is url-encoded + private static void getArraySchema(TileDBClient tileDBClient){ + String namespace = "TileDB-Inc"; // String | namespace array is in (an organization name or user's username) + String array = "quickstart_sparse"; // String | name/uri of array that is url-encoded String contentType = "application/json"; // String | Content Type of input and return mime + ArrayApi apiInstance = new ArrayApi(tileDBClient.getApiClient()); + try { - ArraySchema result = arrayApi.getArray(namespace, array, contentType); + ArraySchema result = apiInstance.getArray(namespace, array, contentType); System.out.println(result); } catch (ApiException e) { System.err.println("Exception when calling ArrayApi#getArray"); @@ -244,7 +263,7 @@ private static void getArraySchema(ArrayApi arrayApi){ } } - private static void createArray(ArrayApi arrayApi){ + private static void createArray(TileDBClient tileDBClient){ String namespace = ""; // String | namespace array is in (an organization name or user's username) String arrayName = "s3:///my_array"; // String | name/uri of array that is url-encoded // String contentType = "application/json"; // String | Content Type of input and return mime @@ -294,8 +313,10 @@ private static void createArray(ArrayApi arrayApi){ schema.setCellOrder(Layout.ROW_MAJOR); System.out.println(schema); + ArrayApi apiInstance = new ArrayApi(tileDBClient.getApiClient()); + try { - arrayApi.createArray(namespace, arrayName, contentType, schema, null); + apiInstance.createArray(namespace, arrayName, contentType, schema, null); } catch (ApiException e) { System.err.println("Exception when calling ArrayApi#createArray"); System.err.println("Status code: " + e.getCode()); @@ -305,14 +326,16 @@ private static void createArray(ArrayApi arrayApi){ } } - public static void registerArray(ArrayApi arrayApi){ + public static void registerArray(TileDBClient tileDBClient){ String namespace = ""; // String | namespace array is in (an organization name or user's username) String array = "s3:////"; // String | name/uri of array that is url-encoded ArrayInfoUpdate arrayMetadata = new ArrayInfoUpdate(); // ArrayInfoUpdate | metadata associated with array arrayMetadata.setUri("s3:////"); arrayMetadata.setName(""); + ArrayApi apiInstance = new ArrayApi(tileDBClient.getApiClient()); + try { - ArrayInfo result = arrayApi.registerArray(namespace, array, arrayMetadata); + ArrayInfo result = apiInstance.registerArray(namespace, array, arrayMetadata); System.out.println(result); } catch (ApiException e) { System.err.println("Exception when calling ArrayApi#registerArray"); diff --git a/src/main/java/io/tiledb/cloud/Pair.java b/src/main/java/io/tiledb/cloud/Pair.java new file mode 100644 index 0000000..f16c7ef --- /dev/null +++ b/src/main/java/io/tiledb/cloud/Pair.java @@ -0,0 +1,39 @@ +package io.tiledb.cloud; + +public class Pair implements java.io.Serializable { + private F first; + private S second; + + private Pair() {} + + public Pair(F first, S second) { + this.first = first; + this.second = second; + } + + public F getFirst() { + return first; + } + + public void setFirst(F first) { + this.first = first; + } + + public S getSecond() { + return second; + } + + public void setSecond(S second) { + this.second = second; + } + + /** + * Returns an empty Pair + * + * @return The Pair + */ + public static Pair empty() { + return new Pair(); + } +} + diff --git a/src/main/java/io/tiledb/cloud/TileDBLogin.java b/src/main/java/io/tiledb/cloud/TileDBLogin.java index 2644bfc..8ebb832 100644 --- a/src/main/java/io/tiledb/cloud/TileDBLogin.java +++ b/src/main/java/io/tiledb/cloud/TileDBLogin.java @@ -1,7 +1,5 @@ package io.tiledb.cloud; -import java.util.Objects; - public class TileDBLogin { /** The client password */ private String password; diff --git a/src/main/java/io/tiledb/cloud/TileDBSQL.java b/src/main/java/io/tiledb/cloud/TileDBSQL.java index 7793e4d..eaf934b 100644 --- a/src/main/java/io/tiledb/cloud/TileDBSQL.java +++ b/src/main/java/io/tiledb/cloud/TileDBSQL.java @@ -2,6 +2,7 @@ import io.tiledb.cloud.rest_api.ApiException; import io.tiledb.cloud.rest_api.api.SqlApi; +import io.tiledb.cloud.rest_api.model.ResultFormat; import io.tiledb.cloud.rest_api.model.SQLParameters; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.memory.UnsafeAllocationManager; @@ -54,9 +55,13 @@ public TileDBSQL(TileDBClient tileDBClient, String namespace, SQLParameters sql) * @return A pair that consists of an ArrayList of all valueVectors and the * number of batches read. */ - public io.tiledb.java.api.Pair, Integer> execArrow(){ + public Pair, Integer> execArrow(){ try { - assert sql.getResultFormat() != null; + if (sql.getResultFormat() != ResultFormat.ARROW && sql.getResultFormat() != null){ + throw new ApiException("The ResultFormat you specified is not 'ARROW'. Since you are calling " + + "'execArrow()' you can not specify a different ResultFormat. "); + } + sql.setResultFormat(ResultFormat.ARROW); byte[] bytes = apiInstance.runSQLBytes(namespace, sql, "none"); ArrayList valueVectors = null; int readBatchesCount = 0; @@ -81,7 +86,7 @@ public io.tiledb.java.api.Pair, Integer> execArrow(){ } } reader.close(); - return new io.tiledb.java.api.Pair<>(valueVectors, readBatchesCount); + return new Pair<>(valueVectors, readBatchesCount); } catch (IOException | ApiException e) { throw new RuntimeException(e); @@ -95,8 +100,11 @@ public io.tiledb.java.api.Pair, Integer> execArrow(){ */ public List exec(){ try { - assert sql.getResultFormat() != null; - return apiInstance.runSQL(namespace, sql, sql.getResultFormat().toString()); + if (sql.getResultFormat() == null ){ + return apiInstance.runSQL(namespace, sql, ResultFormat.TILEDB_JSON.toString()); + } else { + return apiInstance.runSQL(namespace, sql, sql.getResultFormat().toString()); + } } catch (ApiException e) { System.err.println("Exception when calling SqlApi#runSQL/runSQLBytes"); System.err.println("Status code: " + e.getCode()); diff --git a/src/main/java/io/tiledb/cloud/TileDBUDF.java b/src/main/java/io/tiledb/cloud/TileDBUDF.java index d013ca5..1641105 100644 --- a/src/main/java/io/tiledb/cloud/TileDBUDF.java +++ b/src/main/java/io/tiledb/cloud/TileDBUDF.java @@ -1,27 +1,18 @@ package io.tiledb.cloud; -import com.google.gson.Gson; import io.tiledb.cloud.rest_api.ApiException; import io.tiledb.cloud.rest_api.api.UdfApi; -import io.tiledb.cloud.rest_api.model.GenericUDF; -import io.tiledb.cloud.rest_api.model.MultiArrayUDF; -import io.tiledb.cloud.rest_api.model.ResultFormat; -import io.tiledb.java.api.Pair; -import org.apache.arrow.compression.CommonsCompressionFactory; -import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.memory.UnsafeAllocationManager; -import org.apache.arrow.vector.FieldVector; +import io.tiledb.cloud.rest_api.model.*; import org.apache.arrow.vector.ValueVector; -import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.ipc.ArrowStreamReader; -import org.apache.arrow.vector.util.TransferPair; import org.json.JSONArray; import org.json.JSONObject; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.List; + +import static io.tiledb.cloud.TileDBUtils.serializeArgs; public class TileDBUDF { private TileDBClient tileDBClient; @@ -32,7 +23,6 @@ public TileDBUDF(TileDBClient tileDBClient, String namespace) { this.tileDBClient = tileDBClient; this.namespace = namespace; this.apiInstance = new UdfApi(this.tileDBClient.getApiClient()); - } /** @@ -57,6 +47,25 @@ public String executeGeneric(GenericUDF genericUDF, HashMap argu return null; } + /** + * Executes a generic-UDF. A generic-UDF is a UDF that is not using a TIleDB array. + * + * @param genericUDF The generic UDF definition + * @return The result in String format + */ + public String executeGeneric(GenericUDF genericUDF){ + try { + return apiInstance.submitGenericUDFString(namespace, genericUDF, "none"); + } catch (ApiException e) { + System.err.println("Exception when calling UdfApi#submitGenericUDF"); + System.err.println("Status code: " + e.getCode()); + System.err.println("Reason: " + e.getResponseBody()); + System.err.println("Response headers: " + e.getResponseHeaders()); + e.printStackTrace(); + } + return null; + } + /** * Executes a generic-UDF. A generic-UDF is a UDF that is not using a TIleDB array. * @@ -70,6 +79,18 @@ public JSONObject executeGenericJSON(GenericUDF genericUDF, HashMap arguments){ genericUDF.setResultFormat(ResultFormat.JSON); - String jsonString = this.executeGeneric(genericUDF, arguments); + String jsonString = this.executeGeneric(genericUDF, arguments); + return new JSONArray(jsonString); + } + + /** + * Executes a generic-UDF. A generic-UDF is a UDF that is not using a TIleDB array. + * + * @param genericUDF The generic UDF definition + * @return The result as a JSON array object + */ + public JSONArray executeGenericJSONArray(GenericUDF genericUDF){ + genericUDF.setResultFormat(ResultFormat.JSON); + String jsonString = this.executeGeneric(genericUDF); return new JSONArray(jsonString); } @@ -90,7 +123,8 @@ public JSONArray executeGenericJSONArray(GenericUDF genericUDF, HashMap, Integer> executeGenericArrow(GenericUDF genericUDF, HashMap arguments){ + public Pair, Integer> executeGenericArrow(GenericUDF genericUDF, + HashMap arguments){ String serializedArgs = serializeArgs(arguments); genericUDF.setArgument(serializedArgs); genericUDF.setResultFormat(ResultFormat.ARROW); @@ -103,6 +137,24 @@ public io.tiledb.java.api.Pair, Integer> executeGenericAr return null; } + /** + * Executes a generic-UDF. A generic-UDF is a UDF that is not using a TIleDB array. + * + * @param genericUDF The generic UDF definition + * @return A pair that consists of an ArrayList of all valueVectors and the number of batches read. + */ + public Pair, Integer> executeGenericArrow(GenericUDF genericUDF){ + genericUDF.setResultFormat(ResultFormat.ARROW); + try { + byte[] bytes = apiInstance.submitGenericUDFBytes(namespace, genericUDF, "none"); + return TileDBUtils.createValueVectors(bytes); + } catch (IOException | ApiException e) { + e.printStackTrace(); + } + return null; + } + + @Deprecated /** * Executes an array-UDF. An array-UDF is a UDF applied to a TileDB array * @@ -111,6 +163,8 @@ public io.tiledb.java.api.Pair, Integer> executeGenericAr * @param arrayURI The array URI * @param xPayer Name of organization or user who should be charged for this request * @return The results as a String + * @apiNote is replaced by executeSingleArray(GenericUDF genericUDF, UDFArrayDetails array, + * QueryRanges queryRanges, String xPayer) */ public String executeSingleArray(MultiArrayUDF multiArrayUDF, HashMap arguments, String arrayURI, String xPayer){ String serializedArgs = serializeArgs(arguments); @@ -130,6 +184,40 @@ public String executeSingleArray(MultiArrayUDF multiArrayUDF, HashMap arrays = new ArrayList<>(); + arrays.add(array); + + MultiArrayUDF multiArrayUDF = new MultiArrayUDF(); + multiArrayUDF.setArgument(genericUDF.getArgument()); + multiArrayUDF.setUdfInfoName(genericUDF.getUdfInfoName()); + multiArrayUDF.setRanges(queryRanges); + multiArrayUDF.setArrays(arrays); + String[] split = breakdownFullURI(array.getUri()); + + try { + return apiInstance.submitUDFString(split[0], split[1], multiArrayUDF, xPayer, "none", ""); + } catch (ApiException e) { + System.err.println("Exception when calling UdfApi#submitUDF"); + System.err.println("Status code: " + e.getCode()); + System.err.println("Reason: " + e.getResponseBody()); + System.err.println("Response headers: " + e.getResponseHeaders()); + e.printStackTrace(); + } + return null; + } + + @Deprecated /** * Executes an array-UDF. An array-UDF is a UDF applied to a TileDB array * @@ -138,6 +226,8 @@ public String executeSingleArray(MultiArrayUDF multiArrayUDF, HashMap arguments, String arrayURI, String xPayer){ multiArrayUDF.setResultFormat(ResultFormat.JSON); @@ -145,6 +235,42 @@ public JSONObject executeSingleArrayJSON(MultiArrayUDF multiArrayUDF, HashMap arrays = new ArrayList<>(); + arrays.add(array); + + MultiArrayUDF multiArrayUDF = new MultiArrayUDF(); + multiArrayUDF.setArgument(genericUDF.getArgument()); + multiArrayUDF.setUdfInfoName(genericUDF.getUdfInfoName()); + multiArrayUDF.setRanges(queryRanges); + multiArrayUDF.setArrays(arrays); + String[] split = breakdownFullURI(array.getUri()); + multiArrayUDF.setResultFormat(ResultFormat.JSON); + String result = ""; + try { + result = apiInstance.submitUDFString(split[0], split[1], multiArrayUDF, xPayer, "none", ""); + } catch (ApiException e) { + System.err.println("Exception when calling UdfApi#submitUDF"); + System.err.println("Status code: " + e.getCode()); + System.err.println("Reason: " + e.getResponseBody()); + System.err.println("Response headers: " + e.getResponseHeaders()); + e.printStackTrace(); + } + return new JSONObject(result); + } + + @Deprecated /** * Executes an array-UDF. An array-UDF is a UDF applied to a TileDB array * @@ -153,13 +279,51 @@ public JSONObject executeSingleArrayJSON(MultiArrayUDF multiArrayUDF, HashMap arguments, String arrayURI, String xPayer){ + public JSONArray executeSingleArrayJSONArray(MultiArrayUDF multiArrayUDF, HashMap arguments, String arrayURI, String xPayer){ multiArrayUDF.setResultFormat(ResultFormat.JSON); String jsonString = this.executeSingleArray(multiArrayUDF, arguments, arrayURI, xPayer); return new JSONArray(jsonString); } + /** + * Executes an array-UDF. An array-UDF is a UDF applied to a TileDB array + * + * @param genericUDF The UDF to run + * @param array The Array to run the UDF on + * @param queryRanges the Query ranges + * @param xPayer Name of organization or user who should be charged for this request + * @return The results as a JSON Object + */ + public JSONArray executeSingleArrayJSONArray(GenericUDF genericUDF, UDFArrayDetails array, + QueryRanges queryRanges, String xPayer){ + List arrays = new ArrayList<>(); + arrays.add(array); + + MultiArrayUDF multiArrayUDF = new MultiArrayUDF(); + multiArrayUDF.setArgument(genericUDF.getArgument()); + multiArrayUDF.setUdfInfoName(genericUDF.getUdfInfoName()); + multiArrayUDF.setRanges(queryRanges); + multiArrayUDF.setArrays(arrays); + String[] split = breakdownFullURI(array.getUri()); + multiArrayUDF.setResultFormat(ResultFormat.JSON); + String result = ""; + try { + result = apiInstance.submitUDFString(split[0], split[1], multiArrayUDF, xPayer, "none", ""); + } catch (ApiException e) { + System.err.println("Exception when calling UdfApi#submitUDF"); + System.err.println("Status code: " + e.getCode()); + System.err.println("Reason: " + e.getResponseBody()); + System.err.println("Response headers: " + e.getResponseHeaders()); + e.printStackTrace(); + } + return new JSONArray(result); + } + + @Deprecated /** * Executes an array-UDF. An array-UDF is a UDF applied to a TileDB array * @@ -168,6 +332,8 @@ public JSONArray executeSingleArrayJSONArray(MultiArrayUDF multiArrayUDF, HashMa * @param arrayURI The array URI * @param xPayer Name of organization or user who should be charged for this request * @return The results in arrow format + * @apiNote Is replaced by executeSingleArrayArrow(GenericUDF genericUDF, UDFArrayDetails array, + * QueryRanges queryRanges, String xPayer) */ public Pair, Integer> executeSingleArrayArrow(MultiArrayUDF multiArrayUDF, HashMap arguments, String arrayURI, String xPayer){ String serializedArgs = serializeArgs(arguments); @@ -185,6 +351,37 @@ public Pair, Integer> executeSingleArrayArrow(MultiArrayU return null; } + /** + * Executes an array-UDF. An array-UDF is a UDF applied to a TileDB array + * + * @param genericUDF The UDF to run + * @param array The Array to run the UDF on + * @param queryRanges the Query ranges + * @param xPayer Name of organization or user who should be charged for this request + * @return The results in arrow format + */ + public Pair, Integer> executeSingleArrayArrow(GenericUDF genericUDF, UDFArrayDetails array, + QueryRanges queryRanges, String xPayer){ + List arrays = new ArrayList<>(); + arrays.add(array); + + MultiArrayUDF multiArrayUDF = new MultiArrayUDF(); + multiArrayUDF.setArgument(genericUDF.getArgument()); + multiArrayUDF.setResultFormat(ResultFormat.ARROW); + multiArrayUDF.setUdfInfoName(genericUDF.getUdfInfoName()); + multiArrayUDF.setRanges(queryRanges); + multiArrayUDF.setArrays(arrays); + String[] split = breakdownFullURI(array.getUri()); + + try { + byte[] bytes = apiInstance.submitUDFBytes(split[0], split[1], multiArrayUDF, xPayer, "none", ""); + return TileDBUtils.createValueVectors(bytes); + } catch (IOException | ApiException e) { + e.printStackTrace(); + } + return null; + } + /** * Executes a multi-array-UDF. A multi- array-UDF is a UDF applied to multiple TileDB arrays * @@ -207,6 +404,25 @@ public String executeMultiArray(MultiArrayUDF multiArrayUDF, HashMap, Integer> executeMultiArrayArrow(MultiArrayUD } /** - * Serializes the arguments to a String + * Executes a multi-array-UDF. A multi- array-UDF is a UDF applied to multiple TileDB arrays * - * @param arguments The input arguments in a HashMap - * @return The arguments in a String + * @param multiArrayUDF The multiArrayUDF input object + * @return The results in arrow format */ - private String serializeArgs(HashMap arguments) { - if (arguments == null || arguments.isEmpty()) return ""; - Gson gson = new Gson(); - return gson.toJson(arguments); + public Pair, Integer> executeMultiArrayArrow(MultiArrayUDF multiArrayUDF){ + multiArrayUDF.setResultFormat(ResultFormat.ARROW); + + try { + byte[] bytes = apiInstance.submitMultiArrayUDFBytes(this.namespace, multiArrayUDF, "none"); + return TileDBUtils.createValueVectors(bytes); + } catch (IOException | ApiException e) { + e.printStackTrace(); + } + return null; } /** diff --git a/src/main/java/io/tiledb/cloud/TileDBUtils.java b/src/main/java/io/tiledb/cloud/TileDBUtils.java index bf3c2d5..75c3b06 100644 --- a/src/main/java/io/tiledb/cloud/TileDBUtils.java +++ b/src/main/java/io/tiledb/cloud/TileDBUtils.java @@ -1,5 +1,6 @@ package io.tiledb.cloud; +import com.google.gson.Gson; import org.apache.arrow.compression.CommonsCompressionFactory; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.memory.UnsafeAllocationManager; @@ -12,9 +13,22 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; public class TileDBUtils { + /** + * Serializes the arguments to a String + * + * @param arguments The input arguments in a HashMap + * @return The arguments in a String + */ + public static String serializeArgs(HashMap arguments) { + if (arguments == null || arguments.isEmpty()) return ""; + Gson gson = new Gson(); + return gson.toJson(arguments); + } + /** * * This method gets bytes and translated them to arrow valuevectors @@ -22,7 +36,7 @@ public class TileDBUtils { * @return A pair that consists of an ArrayList of all valueVectors and the number of batches read. * @throws IOException */ - static io.tiledb.java.api.Pair, Integer> createValueVectors(byte[] bytes) throws IOException { + protected static Pair, Integer> createValueVectors(byte[] bytes) throws IOException { ArrayList valueVectors = null; int readBatchesCount = 0; @@ -45,6 +59,6 @@ static io.tiledb.java.api.Pair, Integer> createValueVecto } } reader.close(); - return new io.tiledb.java.api.Pair<>(valueVectors, readBatchesCount); + return new Pair<>(valueVectors, readBatchesCount); } } diff --git a/src/main/java/io/tiledb/cloud/rest_api/model/GroupInfo.java b/src/main/java/io/tiledb/cloud/rest_api/model/GroupInfo.java index e634410..a3b2f87 100644 --- a/src/main/java/io/tiledb/cloud/rest_api/model/GroupInfo.java +++ b/src/main/java/io/tiledb/cloud/rest_api/model/GroupInfo.java @@ -659,6 +659,7 @@ private String toIndentedString(Object o) { openapiFields.add("tags"); openapiFields.add("license_id"); openapiFields.add("license_text"); + openapiFields.add("group_type"); // a set of required properties/fields (JSON key names) openapiRequiredFields = new HashSet();