From 5d6550948593ba9019db5ef3f4518f9d9358c964 Mon Sep 17 00:00:00 2001 From: dimitrisstaratzis Date: Fri, 7 Oct 2022 16:57:08 +0100 Subject: [PATCH 1/3] support byte return for sql queries --- .../io/tiledb/cloud/rest_api/api/SqlApi.java | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/main/java/io/tiledb/cloud/rest_api/api/SqlApi.java b/src/main/java/io/tiledb/cloud/rest_api/api/SqlApi.java index bf57393..3a65900 100644 --- a/src/main/java/io/tiledb/cloud/rest_api/api/SqlApi.java +++ b/src/main/java/io/tiledb/cloud/rest_api/api/SqlApi.java @@ -174,6 +174,27 @@ public List runSQL(String namespace, SQLParameters sql, String acceptEnc return localVarResp.getData(); } + /** + * + * Run a sql query + * @param namespace namespace to run task under is in (an organization name or user's username) (required) + * @param sql sql being submitted (required) + * @param acceptEncoding Encoding to use (optional) + * @return byte[] + * @throws ApiException If fail to call the API, e.g. server error or cannot deserialize the response body + * @http.response.details + + + + + +
Status Code Description Response Headers
200 JSON results in array of objects form, if the query returns results * X-TILEDB-CLOUD-TASK-ID - Task ID for just completed request
204 SQL executed successfully * X-TILEDB-CLOUD-TASK-ID - Task ID for just completed request
0 error response -
+ */ + public byte[] runSQLBytes(String namespace, SQLParameters sql, String acceptEncoding) throws ApiException { + ApiResponse localVarResp = runSQLWithHttpInfoBytes(namespace, sql, acceptEncoding); + return localVarResp.getData(); + } + /** * * Run a sql query @@ -196,6 +217,28 @@ public ApiResponse> runSQLWithHttpInfo(String namespace, SQLParamet return localVarApiClient.execute(localVarCall, localVarReturnType); } + /** + * + * Run a sql query + * @param namespace namespace to run task under is in (an organization name or user's username) (required) + * @param sql sql being submitted (required) + * @param acceptEncoding Encoding to use (optional) + * @return ApiResponse with byte[] + * @throws ApiException If fail to call the API, e.g. server error or cannot deserialize the response body + * @http.response.details + + + + + +
Status Code Description Response Headers
200 JSON results in array of objects form, if the query returns results * X-TILEDB-CLOUD-TASK-ID - Task ID for just completed request
204 SQL executed successfully * X-TILEDB-CLOUD-TASK-ID - Task ID for just completed request
0 error response -
+ */ + public ApiResponse runSQLWithHttpInfoBytes(String namespace, SQLParameters sql, String acceptEncoding) throws ApiException { + okhttp3.Call localVarCall = runSQLValidateBeforeCall(namespace, sql, acceptEncoding, null); + Type localVarReturnType = new TypeToken(){}.getType(); + return localVarApiClient.execute(localVarCall, localVarReturnType); + } + /** * (asynchronously) * Run a sql query From 042898c201a8abb9d29ef201db32fd980356355f Mon Sep 17 00:00:00 2001 From: dimitrisstaratzis Date: Mon, 10 Oct 2022 14:57:20 +0100 Subject: [PATCH 2/3] wrap SQL api generated code --- build.gradle | 3 + src/main/java/io/tiledb/cloud/TileDBSQL.java | 121 +++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 src/main/java/io/tiledb/cloud/TileDBSQL.java diff --git a/build.gradle b/build.gradle index 87e58d3..cebdbd3 100644 --- a/build.gradle +++ b/build.gradle @@ -155,10 +155,13 @@ dependencies { implementation group: 'org.apache.oltu.oauth2', name: 'org.apache.oltu.oauth2.client', version: '1.0.1' implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' implementation "jakarta.annotation:jakarta.annotation-api:$jakarta_annotation_version" + implementation group: 'org.apache.arrow', name: 'arrow-vector', version: '9.0.0' + runtimeOnly group: 'org.apache.arrow', name: 'arrow-memory-netty', version: '9.0.0' testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.2' testImplementation 'org.mockito:mockito-core:3.12.4' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.2' testImplementation 'junit:junit:4.13.1' + } import org.gradle.plugins.signing.Sign diff --git a/src/main/java/io/tiledb/cloud/TileDBSQL.java b/src/main/java/io/tiledb/cloud/TileDBSQL.java new file mode 100644 index 0000000..726db17 --- /dev/null +++ b/src/main/java/io/tiledb/cloud/TileDBSQL.java @@ -0,0 +1,121 @@ +package io.tiledb.cloud; + +import io.tiledb.cloud.rest_api.ApiException; +import io.tiledb.cloud.rest_api.api.SqlApi; +import io.tiledb.cloud.rest_api.model.ResultFormat; +import io.tiledb.cloud.rest_api.model.SQLParameters; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.arrow.vector.types.pojo.Schema; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +public class TileDBSQL { + String namespace; + + SQLParameters sql; + + TileDBClient tileDBClient; + + SqlApi apiInstance; + + ArrayList readBatches; + + List results; + + /** + * + * @param tileDBClient The TileDBClient + * @param namespace namespace to run task under is in (an organization name or user's username) + * @param sql sql being submitted + */ + public TileDBSQL(TileDBClient tileDBClient, String namespace, SQLParameters sql) { + Objects.requireNonNull(tileDBClient, "TileDBClient can not be null"); + Objects.requireNonNull(namespace, "Namespace can not be null"); + Objects.requireNonNull(sql, "SQL parameters can not be null"); + this.namespace = namespace; + this.sql = sql; + this.tileDBClient = tileDBClient; + this.apiInstance = new SqlApi(this.tileDBClient.getApiClient()); + this.readBatches = new ArrayList<>(); + } + + /** + * Exec an SQL query and get results in arrow format. + */ + public void execArrow(){ + try { + assert sql.getResultFormat() != null; + byte[] bytes = apiInstance.runSQLBytes(namespace, sql, sql.getResultFormat().toString()); + System.out.println(Arrays.toString(bytes)); + + RootAllocator allocator = new RootAllocator(Long.MAX_VALUE); + try (ArrowStreamReader reader = new ArrowStreamReader(new ByteArrayInputStream(bytes), allocator)) { + while (reader.loadNextBatch()) { + // This will be loaded with new values on every call to loadNextBatch + VectorSchemaRoot readBatch = reader.getVectorSchemaRoot(); + readBatches.add(readBatch); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } catch (ApiException e) { + System.err.println("Exception when calling SqlApi#runSQL/runSQLBytes"); + System.err.println("Status code: " + e.getCode()); + System.err.println("Reason: " + e.getResponseBody()); + System.err.println("Response headers: " + e.getResponseHeaders()); + e.printStackTrace(); + } + } + + /** + * Exec an SQL query and get results in any format except arrow. + * + * @return + */ + public void execStandard(){ + try { + assert sql.getResultFormat() != null; + results = apiInstance.runSQL(namespace, sql, sql.getResultFormat().toString()); + } catch (ApiException e) { + System.err.println("Exception when calling SqlApi#runSQL/runSQLBytes"); + System.err.println("Status code: " + e.getCode()); + System.err.println("Reason: " + e.getResponseBody()); + System.err.println("Response headers: " + e.getResponseHeaders()); + e.printStackTrace(); + } + } + + /** + * Exec an SQL query + */ + public void exec(){ + if (this.sql.getResultFormat() == ResultFormat.ARROW){ + execArrow(); + }else { + execStandard(); + } + } + + /** + * Get the results in Arrow format + * @return + */ + public ArrayList getReadBatches() { + return readBatches; + } + + /** + * Get the results as lists of Objects + * @return + */ + public List getResults() { + return results; + } +} From 47afd714f26d73fff3ba8a0f946ccd48f6bd6553 Mon Sep 17 00:00:00 2001 From: dimitrisstaratzis Date: Mon, 17 Oct 2022 19:25:22 +0300 Subject: [PATCH 3/3] add custom code file --- src/main/java/io/tiledb/cloud/CustomCode.txt | 48 ++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 src/main/java/io/tiledb/cloud/CustomCode.txt diff --git a/src/main/java/io/tiledb/cloud/CustomCode.txt b/src/main/java/io/tiledb/cloud/CustomCode.txt new file mode 100644 index 0000000..f9cb66e --- /dev/null +++ b/src/main/java/io/tiledb/cloud/CustomCode.txt @@ -0,0 +1,48 @@ +This file contains custom methods to add to the generated files. + +SqlApi.java + +/** + * + * Run a sql query + * @param namespace namespace to run task under is in (an organization name or user's username) (required) + * @param sql sql being submitted (required) + * @param acceptEncoding Encoding to use (optional) + * @return ApiResponse with byte[] + * @throws ApiException If fail to call the API, e.g. server error or cannot deserialize the response body + * @http.response.details + + + + + +
Status Code Description Response Headers
200 JSON results in array of objects form, if the query returns results * X-TILEDB-CLOUD-TASK-ID - Task ID for just completed request
204 SQL executed successfully * X-TILEDB-CLOUD-TASK-ID - Task ID for just completed request
0 error response -
+ */ +public ApiResponse runSQLWithHttpInfoBytes(String namespace, SQLParameters sql, String acceptEncoding) throws ApiException { + okhttp3.Call localVarCall = runSQLValidateBeforeCall(namespace, sql, acceptEncoding, null); + Type localVarReturnType = new TypeToken(){}.getType(); + return localVarApiClient.execute(localVarCall, localVarReturnType); +} + +/** + * + * Run a sql query + * @param namespace namespace to run task under is in (an organization name or user's username) (required) + * @param sql sql being submitted (required) + * @param acceptEncoding Encoding to use (optional) + * @return byte[] + * @throws ApiException If fail to call the API, e.g. server error or cannot deserialize the response body + * @http.response.details + + + + + +
Status Code Description Response Headers
200 JSON results in array of objects form, if the query returns results * X-TILEDB-CLOUD-TASK-ID - Task ID for just completed request
204 SQL executed successfully * X-TILEDB-CLOUD-TASK-ID - Task ID for just completed request
0 error response -
+ */ +public byte[] runSQLBytes(String namespace, SQLParameters sql, String acceptEncoding) throws ApiException { + ApiResponse localVarResp = runSQLWithHttpInfoBytes(namespace, sql, acceptEncoding); + return localVarResp.getData(); +} + +