From c3bc04358dbf0280b091477f34c010390473e18f Mon Sep 17 00:00:00 2001 From: Tanmaya Panda <108695755+tanmaya-panda1@users.noreply.github.com> Date: Thu, 27 Oct 2022 12:56:38 +0530 Subject: [PATCH] Switch to jackson library from org.json (#272) * switch to jackson for json * added formatter * fixed failing test cases * fixed testcases * ran formatter * added jackson-core to pom * fixed json string * fixed string conversion * bumped release version * bumped up release version * addressed Ram's review comments * formatter changes * refactored object mapper * addressed Ram's review comments * formatter changes * changes done as per Ohad's comments * added exception testcase for ResultSetTest * added bigdecimal for precision in decimal values * added comment for bigdecimal deserilaization * made options and parameters mandatory in tojson * fixed test case * accept cla * added formatter * commit for cla Co-authored-by: AsafMah --- data/pom.xml | 21 +++- .../azure/kusto/data/ClientImpl.java | 28 ++--- .../kusto/data/ClientRequestProperties.java | 65 +++++----- .../kusto/data/KustoOperationResult.java | 57 +++++++-- .../azure/kusto/data/KustoResultSetTable.java | 119 ++++++++++++------ .../com/microsoft/azure/kusto/data/Utils.java | 32 +++-- .../azure/kusto/data/auth/CloudInfo.java | 28 +++-- .../WellKnownKustoEndpointsData.java | 3 +- .../data/exceptions/DataWebException.java | 17 ++- .../JsonPropertyMissingException.java | 8 ++ .../exceptions/KustoServiceQueryError.java | 13 +- .../kusto/data/exceptions/OneApiError.java | 22 ++-- .../data/ClientRequestPropertiesTest.java | 14 ++- .../azure/kusto/data/KustoDateTimeTest.java | 79 ++++++------ .../azure/kusto/data/ResultSetTest.java | 112 +++++++++++------ ingest/pom.xml | 7 +- .../kusto/ingest/IngestionProperties.java | 7 +- .../ingest/ManagedStreamingIngestClient.java | 11 +- .../kusto/ingest/QueuedIngestClientImpl.java | 7 +- .../microsoft/azure/kusto/ingest/E2ETest.java | 27 ++-- .../kusto/ingest/ResourceManagerTest.java | 16 +-- pom.xml | 5 +- quickstart/pom.xml | 7 +- .../azure/kusto/quickstart/SampleApp.java | 2 +- samples/src/main/java/TableStatus.java | 3 +- 25 files changed, 435 insertions(+), 275 deletions(-) create mode 100644 data/src/main/java/com/microsoft/azure/kusto/data/exceptions/JsonPropertyMissingException.java diff --git a/data/pom.xml b/data/pom.xml index 379fc019..2a135821 100644 --- a/data/pom.xml +++ b/data/pom.xml @@ -170,11 +170,6 @@ httpcore ${httpcore.version} - - org.json - json - ${json.version} - org.jetbrains annotations @@ -282,5 +277,21 @@ com.fasterxml.jackson.core ${fasterxml.jackson.core.version} + + + jackson-core + com.fasterxml.jackson.core + ${fasterxml.jackson.core.version} + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + ${fasterxml.jackson.core.version} + + + com.fasterxml.jackson.datatype + jackson-datatype-jdk8 + ${fasterxml.jackson.core.version} + diff --git a/data/src/main/java/com/microsoft/azure/kusto/data/ClientImpl.java b/data/src/main/java/com/microsoft/azure/kusto/data/ClientImpl.java index 3a3e59af..b0b391e6 100644 --- a/data/src/main/java/com/microsoft/azure/kusto/data/ClientImpl.java +++ b/data/src/main/java/com/microsoft/azure/kusto/data/ClientImpl.java @@ -3,6 +3,8 @@ package com.microsoft.azure.kusto.data; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.azure.kusto.data.auth.CloudInfo; import com.microsoft.azure.kusto.data.auth.ConnectionStringBuilder; import com.microsoft.azure.kusto.data.auth.TokenProviderBase; @@ -16,8 +18,6 @@ import org.apache.http.HttpHeaders; import org.apache.http.client.utils.URIBuilder; import org.apache.http.impl.client.CloseableHttpClient; -import org.json.JSONException; -import org.json.JSONObject; import java.io.InputStream; import java.net.URI; @@ -45,6 +45,8 @@ public class ClientImpl implements Client, StreamingClient { private final CloseableHttpClient httpClient; private boolean endpointValidated = false; + private ObjectMapper objectMapper = Utils.getObjectMapper(); + public ClientImpl(ConnectionStringBuilder csb) throws URISyntaxException { this(csb, HttpClientProperties.builder().build()); } @@ -308,25 +310,17 @@ private Map generateIngestAndCommandHeaders(ClientRequestPropert return headers; } - private String generateCommandPayload(String database, String command, ClientRequestProperties properties, String clusterEndpoint) - throws DataClientException { - String jsonPayload; - try { - JSONObject json = new JSONObject() - .put("db", database) - .put("csl", command); + private String generateCommandPayload(String database, String command, ClientRequestProperties properties, String clusterEndpoint) { - if (properties != null) { - json.put("properties", properties.toString()); - } + ObjectNode json = objectMapper.createObjectNode() + .put("db", database) + .put("csl", command); - jsonPayload = json.toString(); - } catch (JSONException e) { - throw new DataClientException(clusterEndpoint, - String.format(clusterEndpoint, "Error executing command '%s' in database '%s'. Setting up request payload failed.", command, database), e); + if (properties != null) { + json.put("properties", properties.toString()); } - return jsonPayload; + return json.toString(); } private void addCommandHeaders(Map headers) { diff --git a/data/src/main/java/com/microsoft/azure/kusto/data/ClientRequestProperties.java b/data/src/main/java/com/microsoft/azure/kusto/data/ClientRequestProperties.java index 4ad79547..29430a12 100644 --- a/data/src/main/java/com/microsoft/azure/kusto/data/ClientRequestProperties.java +++ b/data/src/main/java/com/microsoft/azure/kusto/data/ClientRequestProperties.java @@ -3,6 +3,9 @@ package com.microsoft.azure.kusto.data; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.azure.kusto.data.format.CslBoolFormat; import com.microsoft.azure.kusto.data.format.CslDateTimeFormat; import com.microsoft.azure.kusto.data.format.CslIntFormat; @@ -12,8 +15,6 @@ import com.microsoft.azure.kusto.data.format.CslUuidFormat; import org.apache.commons.lang3.StringUtils; import org.apache.http.ParseException; -import org.json.JSONException; -import org.json.JSONObject; import java.io.Serializable; import java.time.Duration; @@ -190,57 +191,53 @@ public void setTimeoutInMilliSec(Long timeoutInMs) { options.put(OPTION_SERVER_TIMEOUT, timeoutInMs); } - JSONObject toJson() { - try { - JSONObject optionsAsJSON = new JSONObject(this.options); - Object timeoutObj = getOption(OPTION_SERVER_TIMEOUT); - - if (timeoutObj != null) { - String timeoutString = ""; - if (timeoutObj instanceof Long) { - Duration duration = Duration.ofMillis((Long) timeoutObj); - timeoutString = Utils.formatDurationAsTimespan(duration); - } else if (timeoutObj instanceof String) { - timeoutString = (String) timeoutObj; - } else if (timeoutObj instanceof Integer) { - Duration duration = Duration.ofMillis((Integer) timeoutObj); - timeoutString = Utils.formatDurationAsTimespan(duration); - } - optionsAsJSON.put(OPTION_SERVER_TIMEOUT, timeoutString); + JsonNode toJson() { + ObjectNode optionsAsJSON = Utils.getObjectMapper().valueToTree(this.options); + Object timeoutObj = getOption(OPTION_SERVER_TIMEOUT); + + if (timeoutObj != null) { + String timeoutString = ""; + if (timeoutObj instanceof Long) { + Duration duration = Duration.ofMillis((Long) timeoutObj); + timeoutString = Utils.formatDurationAsTimespan(duration); + } else if (timeoutObj instanceof String) { + timeoutString = (String) timeoutObj; + } else if (timeoutObj instanceof Integer) { + Duration duration = Duration.ofMillis((Integer) timeoutObj); + timeoutString = Utils.formatDurationAsTimespan(duration); } - JSONObject json = new JSONObject(); - json.put(OPTIONS_KEY, optionsAsJSON); - json.put(PARAMETERS_KEY, new JSONObject(this.parameters)); - return json; - } catch (JSONException e) { - return null; + optionsAsJSON.put(OPTION_SERVER_TIMEOUT, timeoutString); } + ObjectNode json = Utils.getObjectMapper().createObjectNode(); + json.set(OPTIONS_KEY, optionsAsJSON); + json.set(PARAMETERS_KEY, Utils.getObjectMapper().valueToTree(this.parameters)); + return json; } public String toString() { return toJson().toString(); } - public static ClientRequestProperties fromString(String json) throws JSONException { + public static ClientRequestProperties fromString(String json) throws JsonProcessingException { if (StringUtils.isNotBlank(json)) { ClientRequestProperties crp = new ClientRequestProperties(); - JSONObject jsonObj = new JSONObject(json); - Iterator it = jsonObj.keys(); + JsonNode jsonObj = Utils.getObjectMapper().readTree(json); + Iterator it = jsonObj.fieldNames(); while (it.hasNext()) { String propertyName = it.next(); if (propertyName.equals(OPTIONS_KEY)) { - JSONObject optionsJson = (JSONObject) jsonObj.get(propertyName); - Iterator optionsIt = optionsJson.keys(); + JsonNode optionsJson = jsonObj.get(propertyName); + Iterator optionsIt = optionsJson.fieldNames(); while (optionsIt.hasNext()) { String optionName = optionsIt.next(); - crp.setOption(optionName, optionsJson.get(optionName)); + crp.setOption(optionName, optionsJson.get(optionName).asText()); } } else if (propertyName.equals(PARAMETERS_KEY)) { - JSONObject parameters = (JSONObject) jsonObj.get(propertyName); - Iterator parametersIt = parameters.keys(); + JsonNode parameters = jsonObj.get(propertyName); + Iterator parametersIt = parameters.fieldNames(); while (parametersIt.hasNext()) { String parameterName = parametersIt.next(); - crp.setParameter(parameterName, parameters.get(parameterName)); + crp.setParameter(parameterName, parameters.get(parameterName).asText()); } } } diff --git a/data/src/main/java/com/microsoft/azure/kusto/data/KustoOperationResult.java b/data/src/main/java/com/microsoft/azure/kusto/data/KustoOperationResult.java index 91b49d09..4e14d0c6 100644 --- a/data/src/main/java/com/microsoft/azure/kusto/data/KustoOperationResult.java +++ b/data/src/main/java/com/microsoft/azure/kusto/data/KustoOperationResult.java @@ -3,13 +3,21 @@ package com.microsoft.azure.kusto.data; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.microsoft.azure.kusto.data.exceptions.JsonPropertyMissingException; import com.microsoft.azure.kusto.data.exceptions.KustoServiceQueryError; -import org.json.JSONArray; -import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.lang.invoke.MethodHandles; import java.util.*; public class KustoOperationResult implements Iterator { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final Map tablesKindsMap = new HashMap() { { put("QueryResult", WellKnownDataSet.PrimaryResult); @@ -26,6 +34,8 @@ public class KustoOperationResult implements Iterator { private final List resultTables = new ArrayList<>(); private final Iterator it; + private final ObjectMapper objectMapper = Utils.getObjectMapper(); + public KustoOperationResult(String response, String version) throws KustoServiceQueryError { if (version.contains("v2")) { createFromV2Response(response); @@ -58,11 +68,20 @@ public KustoResultSetTable getPrimaryResults() { } private void createFromV1Response(String response) throws KustoServiceQueryError { - JSONObject jsonObject = new JSONObject(response); - JSONArray jsonArray = jsonObject.getJSONArray(TABLES_LIST_PROPERTY_NAME); - for (int i = 0; i < jsonArray.length(); i++) { - JSONObject table = jsonArray.getJSONObject(i); - resultTables.add(new KustoResultSetTable(table)); + try { + JsonNode jsonObject = objectMapper.readTree(response); + if (jsonObject.has(TABLES_LIST_PROPERTY_NAME) && jsonObject.get(TABLES_LIST_PROPERTY_NAME).isArray()) { + ArrayNode jsonArray = (ArrayNode) jsonObject.get(TABLES_LIST_PROPERTY_NAME); + for (int i = 0; i < jsonArray.size(); i++) { + JsonNode table = jsonArray.get(i); + resultTables.add(new KustoResultSetTable(table)); + } + } else { + throw new JsonPropertyMissingException("Tables Property missing from V1 response json"); + } + } catch (JsonProcessingException | JsonPropertyMissingException e) { + log.error("Json processing error occured while parsing string to json with exception", e); + throw new KustoServiceQueryError("Json processing error occurred while parsing string to json with exception " + e.getMessage()); } if (resultTables.size() <= 2) { @@ -87,12 +106,26 @@ private void createFromV1Response(String response) throws KustoServiceQueryError } private void createFromV2Response(String response) throws KustoServiceQueryError { - JSONArray jsonArray = new JSONArray(response); - for (int i = 0; i < jsonArray.length(); i++) { - JSONObject table = jsonArray.getJSONObject(i); - if (table.optString(FRAME_TYPE_PROPERTY_NAME).equals(DATA_TABLE_FRAME_TYPE_PROPERTY_NAME)) { - resultTables.add(new KustoResultSetTable(table)); + ArrayNode jsonArray; + try { + JsonNode jsonNode = objectMapper.readTree(response); + if (jsonNode.isArray()) { + jsonArray = (ArrayNode) jsonNode; + for (JsonNode node : jsonArray) { + if (node.has(FRAME_TYPE_PROPERTY_NAME) && node.get(FRAME_TYPE_PROPERTY_NAME).asText().equals(DATA_TABLE_FRAME_TYPE_PROPERTY_NAME)) { + resultTables.add(new KustoResultSetTable(node)); + } + } + } else { + throw new JsonPropertyMissingException("There is no array in the response which can be parsed"); } + } catch (JsonProcessingException | JsonPropertyMissingException jsonException) { + log.error("Json processing error occured while parsing string to json with exception", jsonException); + throw new KustoServiceQueryError( + "Json processing error occurred while parsing string to json with exception " + jsonException.getMessage()); + } catch (NullPointerException nullPointerException) { + log.error("Null pointer exception thrown due to invalid v2 response", nullPointerException); + throw new KustoServiceQueryError("Null pointer exception thrown due to invalid v2 response " + nullPointerException.getMessage()); } } } diff --git a/data/src/main/java/com/microsoft/azure/kusto/data/KustoResultSetTable.java b/data/src/main/java/com/microsoft/azure/kusto/data/KustoResultSetTable.java index 00712eef..63cc8ce3 100644 --- a/data/src/main/java/com/microsoft/azure/kusto/data/KustoResultSetTable.java +++ b/data/src/main/java/com/microsoft/azure/kusto/data/KustoResultSetTable.java @@ -3,12 +3,15 @@ package com.microsoft.azure.kusto.data; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeType; +import com.microsoft.azure.kusto.data.exceptions.JsonPropertyMissingException; import com.microsoft.azure.kusto.data.exceptions.KustoServiceQueryError; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.FastDateFormat; -import org.json.JSONArray; -import org.json.JSONObject; import java.io.ByteArrayInputStream; import java.io.InputStream; @@ -37,7 +40,9 @@ public class KustoResultSetTable { private static final String ROWS_PROPERTY_NAME = "Rows"; private static final String EXCEPTIONS_PROPERTY_NAME = "Exceptions"; private static final String EXCEPTIONS_MESSAGE = "Query execution failed with multiple inner exceptions"; - private static DateTimeFormatter kustoDateTimeFormatter = new DateTimeFormatterBuilder().parseCaseInsensitive() + + private static final String EMPTY_STRING = ""; + private static final DateTimeFormatter kustoDateTimeFormatter = new DateTimeFormatterBuilder().parseCaseInsensitive() .append(DateTimeFormatter.ISO_LOCAL_DATE_TIME).appendLiteral('Z').toFormatter(); private final List> rows; @@ -77,54 +82,96 @@ void setTableKind(WellKnownDataSet tableKind) { this.tableKind = tableKind; } - protected KustoResultSetTable(JSONObject jsonTable) throws KustoServiceQueryError { - tableName = jsonTable.optString(TABLE_NAME_PROPERTY_NAME); - tableId = jsonTable.optString(TABLE_ID_PROPERTY_NAME); - String tableKindString = jsonTable.optString(TABLE_KIND_PROPERTY_NAME); + protected KustoResultSetTable(JsonNode jsonTable) throws KustoServiceQueryError, JsonProcessingException, JsonPropertyMissingException { + if (jsonTable.has(TABLE_NAME_PROPERTY_NAME)) { + tableName = jsonTable.get(TABLE_NAME_PROPERTY_NAME).asText(); + } + if (jsonTable.has(TABLE_ID_PROPERTY_NAME)) { + tableId = jsonTable.get(TABLE_ID_PROPERTY_NAME).asText(); + } + tableId = jsonTable.has(TABLE_ID_PROPERTY_NAME) ? jsonTable.get(TABLE_ID_PROPERTY_NAME).asText() : EMPTY_STRING; + String tableKindString = jsonTable.has(TABLE_KIND_PROPERTY_NAME) ? jsonTable.get(TABLE_KIND_PROPERTY_NAME).asText() : EMPTY_STRING; tableKind = StringUtils.isBlank(tableKindString) ? null : WellKnownDataSet.valueOf(tableKindString); - JSONArray columnsJson = jsonTable.optJSONArray(COLUMNS_PROPERTY_NAME); - if (columnsJson != null) { - columnsAsArray = new KustoResultColumn[columnsJson.length()]; - for (int i = 0; i < columnsJson.length(); i++) { - JSONObject jsonCol = columnsJson.getJSONObject(i); - String columnType = jsonCol.optString(COLUMN_TYPE_PROPERTY_NAME); - if (columnType.equals("")) { - columnType = jsonCol.optString(COLUMN_TYPE_SECOND_PROPERTY_NAME); + + if (jsonTable.has(COLUMNS_PROPERTY_NAME) && jsonTable.get(COLUMNS_PROPERTY_NAME).getNodeType() == JsonNodeType.ARRAY) { + ArrayNode columnsJson = (ArrayNode) jsonTable.get(COLUMNS_PROPERTY_NAME); + if (columnsJson != null) { + columnsAsArray = new KustoResultColumn[columnsJson.size()]; + for (int i = 0; i < columnsJson.size(); i++) { + JsonNode jsonCol = columnsJson.get(i); + + String columnType = jsonCol.has(COLUMN_TYPE_PROPERTY_NAME) ? jsonCol.get(COLUMN_TYPE_PROPERTY_NAME).asText() : EMPTY_STRING; + if (columnType.equals("")) { + columnType = jsonCol.has(COLUMN_TYPE_SECOND_PROPERTY_NAME) ? jsonCol.get(COLUMN_TYPE_SECOND_PROPERTY_NAME).asText() : EMPTY_STRING; + } + if (jsonCol.has(COLUMN_NAME_PROPERTY_NAME)) { + KustoResultColumn col = new KustoResultColumn( + jsonCol.has(COLUMN_NAME_PROPERTY_NAME) ? jsonCol.get(COLUMN_NAME_PROPERTY_NAME).asText() : EMPTY_STRING, columnType, i); + columnsAsArray[i] = col; + columns.put(jsonCol.has(COLUMN_NAME_PROPERTY_NAME) ? jsonCol.get(COLUMN_NAME_PROPERTY_NAME).asText() : EMPTY_STRING, col); + } else { + throw new JsonPropertyMissingException("Column Name property is missing in the json response"); + } } - KustoResultColumn col = new KustoResultColumn(jsonCol.getString(COLUMN_NAME_PROPERTY_NAME), columnType, i); - columnsAsArray[i] = col; - columns.put(jsonCol.getString(COLUMN_NAME_PROPERTY_NAME), col); } } - JSONArray exceptions; - JSONArray jsonRows = jsonTable.optJSONArray(ROWS_PROPERTY_NAME); + ArrayNode exceptions; + ArrayNode jsonRows = null; + if (jsonTable.has(ROWS_PROPERTY_NAME) && jsonTable.get(ROWS_PROPERTY_NAME).getNodeType() == JsonNodeType.ARRAY) { + jsonRows = (ArrayNode) jsonTable.get(ROWS_PROPERTY_NAME); + } if (jsonRows != null) { List> values = new ArrayList<>(); - for (int i = 0; i < jsonRows.length(); i++) { - Object row = jsonRows.get(i); - if (row instanceof JSONObject) { - exceptions = ((JSONObject) row).optJSONArray(EXCEPTIONS_PROPERTY_NAME); + for (int i = 0; i < jsonRows.size(); i++) { + JsonNode row = jsonRows.get(i); + if (jsonRows.get(i).getNodeType() == JsonNodeType.OBJECT) { + exceptions = row.has(EXCEPTIONS_PROPERTY_NAME) ? ((ArrayNode) row.get(EXCEPTIONS_PROPERTY_NAME)) : null; if (exceptions != null) { - if (exceptions.length() == 1) { - String message = exceptions.getString(0); + if (exceptions.size() == 1) { + String message = exceptions.get(0).asText(); throw new KustoServiceQueryError(message); } else { throw new KustoServiceQueryError(exceptions, false, EXCEPTIONS_MESSAGE); } } else { - throw new KustoServiceQueryError(((JSONObject) row).getJSONArray( - "OneApiErrors"), true, EXCEPTIONS_MESSAGE); + throw new KustoServiceQueryError((ArrayNode) row.get("OneApiErrors"), true, EXCEPTIONS_MESSAGE); } } - JSONArray rowAsJsonArray = jsonRows.getJSONArray(i); + ArrayNode rowAsJsonArray = (ArrayNode) jsonRows.get(i); List rowVector = new ArrayList<>(); - for (int j = 0; j < rowAsJsonArray.length(); ++j) { - Object obj = rowAsJsonArray.get(j); - if (obj == JSONObject.NULL) { + for (int j = 0; j < rowAsJsonArray.size(); j++) { + JsonNode obj = rowAsJsonArray.get(j); + if (obj.isNull()) { rowVector.add(null); } else { - rowVector.add(obj); + switch (rowAsJsonArray.get(j).getNodeType()) { + case STRING: + rowVector.add(obj.asText()); + break; + case BOOLEAN: + rowVector.add(obj.asBoolean()); + break; + case NUMBER: + if (obj.isInt()) { + rowVector.add(obj.asInt()); + } else if (obj.isLong()) { + rowVector.add(obj.asLong()); + } else if (obj.isBigDecimal()) { + rowVector.add(obj.decimalValue()); + } else if (obj.isDouble()) { + rowVector.add(obj.asDouble()); + } else if (obj.isShort()) { + rowVector.add(obj.shortValue()); + } else if (obj.isFloat()) { + rowVector.add(obj.floatValue()); + } else { + rowVector.add(obj); + } + break; + default: + rowVector.add(obj); + } } } values.add(rowVector); @@ -371,12 +418,12 @@ public Object getObject(String columnName) { return get(columnName); } - public JSONObject getJSONObject(String colName) { + public JsonNode getJSONObject(String colName) { return getJSONObject(findColumn(colName)); } - public JSONObject getJSONObject(int columnIndex) { - return (JSONObject) get(columnIndex); + public JsonNode getJSONObject(int columnIndex) { + return (JsonNode) get(columnIndex); } public UUID getUUID(int columnIndex) { diff --git a/data/src/main/java/com/microsoft/azure/kusto/data/Utils.java b/data/src/main/java/com/microsoft/azure/kusto/data/Utils.java index 6a0293c7..0e6925d3 100644 --- a/data/src/main/java/com/microsoft/azure/kusto/data/Utils.java +++ b/data/src/main/java/com/microsoft/azure/kusto/data/Utils.java @@ -3,10 +3,19 @@ package com.microsoft.azure.kusto.data; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.microsoft.azure.kusto.data.auth.CloudInfo; import com.microsoft.azure.kusto.data.exceptions.*; -import com.microsoft.azure.kusto.data.auth.endpoints.KustoTrustedEndpoints; import org.apache.commons.lang3.StringUtils; import org.apache.http.Header; import org.apache.http.HttpEntity; @@ -24,8 +33,6 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.util.EntityUtils; import org.jetbrains.annotations.NotNull; -import org.json.JSONException; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,10 +53,18 @@ import java.util.zip.DeflaterInputStream; import java.util.zip.GZIPInputStream; -class Utils { +public class Utils { private static final int MAX_REDIRECT_COUNT = 1; private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + // added auto bigdecimal deserialization for float and double value, since the bigdecimal values seem to loose precision while auto deserialization to + // double value + public static ObjectMapper getObjectMapper() { + return JsonMapper.builder().addModule(new JavaTimeModule()).addModule(new Jdk8Module()).build().configure( + DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, true).configure(JsonGenerator.Feature.WRITE_BIGDECIMAL_AS_PLAIN, true).setNodeFactory( + JsonNodeFactory.withExactBigDecimals(true)); + } + private Utils() { // Hide constructor, as this is a static utility class } @@ -83,7 +98,7 @@ static String post(CloseableHttpClient httpClient, String urlStr, String payload } } catch (SocketTimeoutException e) { throw new DataServiceException(urlStr, "Timed out in post request:" + e.getMessage(), false); - } catch (JSONException | IOException e) { + } catch (IOException e) { throw new DataClientException(urlStr, "Error in post request:" + e.getMessage(), e); } return null; @@ -177,17 +192,18 @@ static DataServiceException createExceptionFromResponse(String url, HttpResponse boolean isPermanent = false; if (!StringUtils.isBlank(errorFromResponse)) { try { - JSONObject jsonObject = new JSONObject(errorFromResponse); + JsonNode jsonObject = getObjectMapper().readTree(errorFromResponse); if (jsonObject.has("error")) { formattedException = new DataWebException(errorFromResponse, httpResponse, thrownException); OneApiError apiError = ((DataWebException) formattedException).getApiError(); message = apiError.getDescription(); isPermanent = apiError.isPermanent(); } else if (jsonObject.has("message")) { - message = jsonObject.getString("message"); + message = jsonObject.get("message").asText(); } - } catch (JSONException ex) { + } catch (JsonProcessingException e) { // It's not ideal to use an exception here for control flow, but we can't know if it's a valid JSON until we try to parse it + LOGGER.debug("json processing error happened while parsing errorFromResponse {} {}" + e.getMessage(), e); } } else { message = String.format("Http StatusCode='%s'", httpResponse.getStatusLine().toString()); diff --git a/data/src/main/java/com/microsoft/azure/kusto/data/auth/CloudInfo.java b/data/src/main/java/com/microsoft/azure/kusto/data/auth/CloudInfo.java index d180bdce..3aee4541 100644 --- a/data/src/main/java/com/microsoft/azure/kusto/data/auth/CloudInfo.java +++ b/data/src/main/java/com/microsoft/azure/kusto/data/auth/CloudInfo.java @@ -1,7 +1,11 @@ package com.microsoft.azure.kusto.data.auth; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.azure.kusto.data.UriUtils; +import com.microsoft.azure.kusto.data.Utils; import com.microsoft.azure.kusto.data.exceptions.DataServiceException; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpHeaders; @@ -11,7 +15,6 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import org.jetbrains.annotations.Nullable; -import org.json.JSONObject; import java.io.Closeable; import java.io.IOException; @@ -126,19 +129,22 @@ public static CloudInfo retrieveCloudInfoForCluster(String clusterUrl, } } - private static CloudInfo parseCloudInfo(String content) { - JSONObject jsonObject = new JSONObject(content); - JSONObject innerObject = jsonObject.optJSONObject("AzureAD"); + private static CloudInfo parseCloudInfo(String content) throws JsonProcessingException { + ObjectMapper objectMapper = Utils.getObjectMapper(); + JsonNode jsonObject = objectMapper.readTree(content); + JsonNode innerObject = jsonObject.has("AzureAD") ? jsonObject.get("AzureAD") : null; if (innerObject == null) { return DEFAULT_CLOUD; + } else { + return new CloudInfo( + innerObject.has("LoginMfaRequired") && innerObject.get("LoginMfaRequired").asBoolean(), + innerObject.has("LoginEndpoint") ? innerObject.get("LoginEndpoint").asText() : "", + innerObject.has("KustoClientAppId") ? innerObject.get("KustoClientAppId").asText() : "", + innerObject.has("KustoClientRedirectUri") ? innerObject.get("KustoClientRedirectUri").asText() : "", + innerObject.has("KustoServiceResourceId") ? innerObject.get("KustoServiceResourceId").asText() : "", + innerObject.has("FirstPartyAuthorityUrl") ? innerObject.get("FirstPartyAuthorityUrl").asText() : ""); } - return new CloudInfo( - innerObject.getBoolean("LoginMfaRequired"), - innerObject.getString("LoginEndpoint"), - innerObject.getString("KustoClientAppId"), - innerObject.getString("KustoClientRedirectUri"), - innerObject.getString("KustoServiceResourceId"), - innerObject.getString("FirstPartyAuthorityUrl")); + } @Override diff --git a/data/src/main/java/com/microsoft/azure/kusto/data/auth/endpoints/WellKnownKustoEndpointsData.java b/data/src/main/java/com/microsoft/azure/kusto/data/auth/endpoints/WellKnownKustoEndpointsData.java index 42180ad8..10bd2f78 100644 --- a/data/src/main/java/com/microsoft/azure/kusto/data/auth/endpoints/WellKnownKustoEndpointsData.java +++ b/data/src/main/java/com/microsoft/azure/kusto/data/auth/endpoints/WellKnownKustoEndpointsData.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.azure.kusto.data.Utils; import java.io.InputStream; import java.util.ArrayList; @@ -37,7 +38,7 @@ public WellKnownKustoEndpointsData() { private static WellKnownKustoEndpointsData readInstance() { try { // Beautiful ! - ObjectMapper objectMapper = new ObjectMapper(); + ObjectMapper objectMapper = Utils.getObjectMapper(); try (InputStream resourceAsStream = WellKnownKustoEndpointsData.class.getResourceAsStream( "/WellKnownKustoEndpoints.json")) { return objectMapper.readValue(resourceAsStream, WellKnownKustoEndpointsData.class); diff --git a/data/src/main/java/com/microsoft/azure/kusto/data/exceptions/DataWebException.java b/data/src/main/java/com/microsoft/azure/kusto/data/exceptions/DataWebException.java index dd0b01ff..dcaf1e13 100644 --- a/data/src/main/java/com/microsoft/azure/kusto/data/exceptions/DataWebException.java +++ b/data/src/main/java/com/microsoft/azure/kusto/data/exceptions/DataWebException.java @@ -3,12 +3,21 @@ package com.microsoft.azure.kusto.data.exceptions; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.azure.kusto.data.Utils; import org.apache.http.HttpResponse; -import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.invoke.MethodHandles; public class DataWebException extends WebException { private OneApiError apiError = null; + private final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private ObjectMapper objectMapper = Utils.getObjectMapper(); + public DataWebException(String message, HttpResponse httpResponse, Throwable cause) { super(message, httpResponse, cause); } @@ -23,7 +32,11 @@ public DataWebException(String message) { public OneApiError getApiError() { if (apiError == null) { - apiError = OneApiError.fromJsonObject(new JSONObject(getMessage()).getJSONObject("error")); + try { + apiError = OneApiError.fromJsonObject(objectMapper.readTree(getMessage()).get("error")); + } catch (JsonProcessingException e) { + log.error("failed to parse error from message {} {} ", e.getMessage(), e); + } } return apiError; } diff --git a/data/src/main/java/com/microsoft/azure/kusto/data/exceptions/JsonPropertyMissingException.java b/data/src/main/java/com/microsoft/azure/kusto/data/exceptions/JsonPropertyMissingException.java new file mode 100644 index 00000000..a8a3de7d --- /dev/null +++ b/data/src/main/java/com/microsoft/azure/kusto/data/exceptions/JsonPropertyMissingException.java @@ -0,0 +1,8 @@ +package com.microsoft.azure.kusto.data.exceptions; + +public class JsonPropertyMissingException extends Exception { + + public JsonPropertyMissingException(String message) { + super(message); + } +} diff --git a/data/src/main/java/com/microsoft/azure/kusto/data/exceptions/KustoServiceQueryError.java b/data/src/main/java/com/microsoft/azure/kusto/data/exceptions/KustoServiceQueryError.java index f53c8093..a4e50d4d 100644 --- a/data/src/main/java/com/microsoft/azure/kusto/data/exceptions/KustoServiceQueryError.java +++ b/data/src/main/java/com/microsoft/azure/kusto/data/exceptions/KustoServiceQueryError.java @@ -3,8 +3,9 @@ package com.microsoft.azure.kusto.data.exceptions; -import org.json.JSONArray; -import org.json.JSONException; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.node.ArrayNode; import java.util.ArrayList; import java.util.List; @@ -15,14 +16,14 @@ public class KustoServiceQueryError extends Exception { private final List exceptions; - public KustoServiceQueryError(JSONArray jsonExceptions, boolean isOneApi, String message) throws JSONException { + public KustoServiceQueryError(ArrayNode jsonExceptions, boolean isOneApi, String message) { super(message); this.exceptions = new ArrayList<>(); - for (int j = 0; j < jsonExceptions.length(); j++) { + for (int j = 0; jsonExceptions != null && j < jsonExceptions.size(); j++) { if (isOneApi) { - this.exceptions.add(new DataWebException(jsonExceptions.getJSONObject(j).toString())); + this.exceptions.add(new DataWebException(jsonExceptions.get(j).toString())); } else { - this.exceptions.add(new Exception(jsonExceptions.getString(j))); + this.exceptions.add(new Exception(jsonExceptions.get(j).toString())); } } } diff --git a/data/src/main/java/com/microsoft/azure/kusto/data/exceptions/OneApiError.java b/data/src/main/java/com/microsoft/azure/kusto/data/exceptions/OneApiError.java index 2862de65..1bea951a 100644 --- a/data/src/main/java/com/microsoft/azure/kusto/data/exceptions/OneApiError.java +++ b/data/src/main/java/com/microsoft/azure/kusto/data/exceptions/OneApiError.java @@ -3,10 +3,10 @@ package com.microsoft.azure.kusto.data.exceptions; -import org.json.JSONObject; +import com.fasterxml.jackson.databind.JsonNode; public class OneApiError { - public OneApiError(String code, String message, String description, String type, JSONObject context, boolean permanent) { + public OneApiError(String code, String message, String description, String type, JsonNode context, boolean permanent) { this.code = code; this.message = message; this.description = description; @@ -15,21 +15,21 @@ public OneApiError(String code, String message, String description, String type, this.permanent = permanent; } - public static OneApiError fromJsonObject(JSONObject jsonObject) { + public static OneApiError fromJsonObject(JsonNode jsonObject) { return new OneApiError( - jsonObject.getString("code"), - jsonObject.getString("message"), - jsonObject.getString("@message"), - jsonObject.getString("@type"), - jsonObject.getJSONObject("@context"), - jsonObject.getBoolean("@permanent")); + jsonObject.has("code") ? jsonObject.get("code").asText() : "", + jsonObject.has("message") ? jsonObject.get("message").asText() : "", + jsonObject.has("@message") ? jsonObject.get("@message").asText() : "", + jsonObject.has("@type") ? jsonObject.get("@type").asText() : "", + jsonObject.get("@context"), + jsonObject.has("@permanent") && jsonObject.get("@permanent").asBoolean()); } private final String code; private final String message; private final String description; private final String type; - private final JSONObject context; + private final JsonNode context; private final boolean permanent; public String getCode() { @@ -48,7 +48,7 @@ public String getType() { return type; } - public JSONObject getContext() { + public JsonNode getContext() { return context; } diff --git a/data/src/test/java/com/microsoft/azure/kusto/data/ClientRequestPropertiesTest.java b/data/src/test/java/com/microsoft/azure/kusto/data/ClientRequestPropertiesTest.java index 0d4c8ed2..292587a0 100644 --- a/data/src/test/java/com/microsoft/azure/kusto/data/ClientRequestPropertiesTest.java +++ b/data/src/test/java/com/microsoft/azure/kusto/data/ClientRequestPropertiesTest.java @@ -3,13 +3,13 @@ package com.microsoft.azure.kusto.data; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.azure.kusto.data.format.CslDateTimeFormat; import com.microsoft.azure.kusto.data.format.CslTimespanFormat; -import org.json.JSONException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -import org.skyscreamer.jsonassert.JSONAssert; import java.time.Duration; import java.time.LocalDateTime; @@ -34,21 +34,23 @@ void timeoutSetGet() { @Test @DisplayName("test ClientRequestProperties toString") - void propertiesToString() throws JSONException { + void propertiesToString() throws JsonProcessingException { ClientRequestProperties props = new ClientRequestProperties(); props.setOption("a", 1); props.setOption("b", "hello"); + ObjectMapper objectMapper = Utils.getObjectMapper(); - JSONAssert.assertEquals("{\"Options\": {\"a\":1, \"b\":\"hello\"}}", props.toString(), false); + Assertions.assertEquals(objectMapper.readTree("{\"Options\":{\"a\":1,\"b\":\"hello\"},\"Parameters\":{}}").toString(), props.toString()); } @Test @DisplayName("test ClientRequestProperties fromString") - void stringToProperties() throws JSONException { + void stringToProperties() throws JsonProcessingException { String properties = "{\"Options\":{\"servertimeout\":\"01:25:11.111\", \"Content-Encoding\":\"gzip\"},\"Parameters\":{\"birthday\":\"datetime(1970-05-11)\",\"courses\":\"dynamic(['Java', 'C++'])\"}}"; ClientRequestProperties crp = ClientRequestProperties.fromString(properties); + assert crp != null; - assert crp.toJson().getJSONObject("Options").get("servertimeout").equals("01:25:11.111"); + assert crp.toJson().get("Options").get("servertimeout").asText().equals("01:25:11.111"); assert crp.getTimeoutInMilliSec() != null; assert crp.getOption("Content-Encoding").equals("gzip"); assert crp.getParameter("birthday").equals("datetime(1970-05-11)"); diff --git a/data/src/test/java/com/microsoft/azure/kusto/data/KustoDateTimeTest.java b/data/src/test/java/com/microsoft/azure/kusto/data/KustoDateTimeTest.java index 09f06e9e..1e3cdfd2 100644 --- a/data/src/test/java/com/microsoft/azure/kusto/data/KustoDateTimeTest.java +++ b/data/src/test/java/com/microsoft/azure/kusto/data/KustoDateTimeTest.java @@ -1,58 +1,63 @@ package com.microsoft.azure.kusto.data; -import org.json.JSONArray; -import org.json.JSONObject; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.time.Instant; import java.time.LocalDateTime; -import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; public class KustoDateTimeTest { + @Test void KustoResultSet() throws Exception { - JSONArray rows = new JSONArray(); - JSONArray row = new JSONArray(); - row.put(Instant.parse("2022-05-17T00:00:00Z")); - rows.put(row); - row = new JSONArray(); - row.put(Instant.parse("2022-05-17T00:00:00.1Z")); - rows.put(row); - row = new JSONArray(); - row.put(Instant.parse("2022-05-17T00:00:00.12Z")); - rows.put(row); - row = new JSONArray(); - row.put(Instant.parse("2022-05-17T00:00:00.123Z")); - rows.put(row); - row = new JSONArray(); - row.put(Instant.parse("2022-05-17T00:00:00.1234Z")); - rows.put(row); - row = new JSONArray(); - row.put(Instant.parse("2022-05-17T00:00:00.12345Z")); - rows.put(row); - row = new JSONArray(); - row.put(Instant.parse("2022-05-17T00:00:00.123456Z")); - rows.put(row); - row = new JSONArray(); - row.put(Instant.parse("2022-05-17T00:00:00.1234567Z")); - rows.put(row); - row = new JSONArray(); - row.put(Instant.parse("2022-05-17T00:00:00.12345678Z")); - rows.put(row); - row = new JSONArray(); - row.put(Instant.parse("2022-05-17T00:00:00.123456789Z")); - rows.put(row); + ObjectMapper mapper = Utils.getObjectMapper(); + DateTimeFormatter kustoDateTimeFormatter = new DateTimeFormatterBuilder().parseCaseInsensitive() + .append(DateTimeFormatter.ISO_LOCAL_DATE_TIME).appendLiteral('Z').toFormatter(); + ArrayNode rows = mapper.createArrayNode(); + ArrayNode row = mapper.createArrayNode(); + row.add(Instant.parse("2022-05-17T00:00:00Z").toString()); + rows.add(row); + row = mapper.createArrayNode(); + row.add(Instant.parse("2022-05-17T00:00:00.1Z").toString()); + rows.add(row); + row = mapper.createArrayNode(); + row.add(Instant.parse("2022-05-17T00:00:00.12Z").toString()); + rows.add(row); + row = mapper.createArrayNode(); + row.add(Instant.parse("2022-05-17T00:00:00.123Z").toString()); + rows.add(row); + row = mapper.createArrayNode(); + row.add(Instant.parse("2022-05-17T00:00:00.1234Z").toString()); + rows.add(row); + row = mapper.createArrayNode(); + row.add(Instant.parse("2022-05-17T00:00:00.12345Z").toString()); + rows.add(row); + row = mapper.createArrayNode(); + row.add(Instant.parse("2022-05-17T00:00:00.123456Z").toString()); + rows.add(row); + row = mapper.createArrayNode(); + row.add(Instant.parse("2022-05-17T00:00:00.1234567Z").toString()); + rows.add(row); + row = mapper.createArrayNode(); + row.add(Instant.parse("2022-05-17T00:00:00.12345678Z").toString()); + rows.add(row); + row = mapper.createArrayNode(); + row.add(Instant.parse("2022-05-17T00:00:00.123456789Z").toString()); + rows.add(row); String columns = "[ { \"ColumnName\": \"a\", \"ColumnType\": \"datetime\" } ]"; - KustoResultSetTable res = new KustoResultSetTable(new JSONObject("{\"TableName\":\"Table_0\"," + + KustoResultSetTable res = new KustoResultSetTable(mapper.createArrayNode().add("{\"TableName\":\"Table_0\"," + "\"Columns\":" + columns + ",\"Rows\":" + - rows.toString() + "}")); + rows + "}")); Integer rowNum = 0; while (res.next()) { Assertions.assertEquals( - LocalDateTime.ofInstant((Instant) ((JSONArray) rows.get(rowNum)).get(0), ZoneOffset.UTC), + LocalDateTime.parse(rows.get(rowNum).get(0).toString(), kustoDateTimeFormatter), res.getKustoDateTime(0)); rowNum++; } diff --git a/data/src/test/java/com/microsoft/azure/kusto/data/ResultSetTest.java b/data/src/test/java/com/microsoft/azure/kusto/data/ResultSetTest.java index bdaf97c0..348d07df 100644 --- a/data/src/test/java/com/microsoft/azure/kusto/data/ResultSetTest.java +++ b/data/src/test/java/com/microsoft/azure/kusto/data/ResultSetTest.java @@ -1,7 +1,10 @@ package com.microsoft.azure.kusto.data; -import org.json.JSONArray; -import org.json.JSONObject; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.microsoft.azure.kusto.data.exceptions.KustoServiceQueryError; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -22,22 +25,25 @@ public class ResultSetTest { @Test void KustoResultSet() throws Exception { - JSONArray rows = new JSONArray(); - JSONArray row1 = new JSONArray(); - JSONArray row2 = new JSONArray(); - JSONObject nullObj = null; - row2.put(nullObj); - row2.put(""); - row2.put(nullObj); - row2.put(nullObj); - row2.put(nullObj); - row2.put(nullObj); - row2.put(nullObj); - row2.put(nullObj); - row2.put(nullObj); - row2.put(nullObj); - row2.put(nullObj); - rows.put(row2); + ObjectMapper objectMapper = Utils.getObjectMapper(); + + ArrayNode rows = objectMapper.createArrayNode(); + ArrayNode row1 = objectMapper.createArrayNode(); + ArrayNode row2 = objectMapper.createArrayNode(); + + JsonNode nullObj = null; + row2.add(nullObj); + row2.add(""); + row2.add(nullObj); + row2.add(nullObj); + row2.add(nullObj); + row2.add(nullObj); + row2.add(nullObj); + row2.add(nullObj); + row2.add(nullObj); + row2.add(nullObj); + row2.add(nullObj); + rows.add(row2); String str = "str"; Instant now = Instant.now(); @@ -49,19 +55,23 @@ void KustoResultSet() throws Exception { long l = 100000000000L; double d = 1.1d; short s1 = 10; + float f1 = 15.0f; + BigDecimal bigDecimal = new BigDecimal("10.0003214134245341414141314134134101"); String durationAsKustoString = LocalTime.MIDNIGHT.plus(duration).toString(); - row1.put(true); - row1.put(str); - row1.put(now); - row1.put(dec); - row1.put(new JSONObject()); - row1.put(uuid); - row1.put(i); - row1.put(l); - row1.put(BigDecimal.valueOf(d)); - row1.put(durationAsKustoString); - row1.put(s1); - rows.put(row1); + row1.add(true); + row1.add(str); + row1.add(String.valueOf(now)); + row1.add(dec); + row1.add(objectMapper.createObjectNode()); + row1.add(String.valueOf(uuid)); + row1.add(i); + row1.add(l); + row1.add(bigDecimal); + row1.add(durationAsKustoString); + row1.add(s1); + row1.add(f1); + row1.add(d); + rows.add(row1); String columns = "[ { \"ColumnName\": \"a\", \"ColumnType\": \"bool\" }, { \"ColumnName\": \"b\", " + "\"ColumnType\": \"string\" }, { \"ColumnName\": \"c\", \"ColumnType\": \"datetime\" }, { " + @@ -70,9 +80,9 @@ void KustoResultSet() throws Exception { "\"ColumnName\": \"g\", \"ColumnType\": \"int\" }, { \"ColumnName\": \"h\", \"ColumnType\": " + "\"long\" }, { \"ColumnName\": \"i\", \"ColumnType\": \"real\" }, { \"ColumnName\": \"j\", " + "\"ColumnType\": \"timespan\" },{ \"ColumnName\": \"k\", \"ColumnType\": \"short\" } ]"; - KustoResultSetTable res = new KustoResultSetTable(new JSONObject("{\"TableName\":\"Table_0\"," + + KustoResultSetTable res = new KustoResultSetTable(objectMapper.readTree("{\"TableName\":\"Table_0\"," + "\"Columns\":" + columns + ",\"Rows\":" + - rows.toString() + "}")); + rows + "}")); res.next(); assert res.getBooleanObject(0) == null; assert res.getString(1).equals(""); @@ -85,7 +95,7 @@ void KustoResultSet() throws Exception { assert res.getLongObject(7) == null; assert res.getDoubleObject(8) == null; assert res.getTime(9) == null; - assert res.getShortObject(9) == null; + assert res.getShortObject(10) == null; res.next(); Assertions.assertTrue(res.getBooleanObject(0)); @@ -97,14 +107,46 @@ void KustoResultSet() throws Exception { Assertions.assertEquals(sdf.format(new Date(now.getEpochSecond() * 1000)), res.getDate(2).toString()); Assertions.assertEquals(res.getBigDecimal(3), dec); - Assertions.assertEquals(res.getJSONObject(4).toString(), new JSONObject().toString()); + Assertions.assertEquals(res.getJSONObject(4), objectMapper.createObjectNode()); Assertions.assertEquals(res.getUUID(5), uuid); Assertions.assertEquals(res.getIntegerObject(6), i); Assertions.assertEquals(res.getLongObject(7), l); - Assertions.assertEquals(res.getDoubleObject(8), d); + Assertions.assertEquals(res.getBigDecimal(8), bigDecimal); Assertions.assertEquals(res.getTime(9), Time.valueOf(durationAsKustoString)); Assertions.assertEquals(res.getLocalTime(9), LocalTime.parse(durationAsKustoString)); Assertions.assertEquals(res.getShort(10), s1); + Assertions.assertEquals(res.getBigDecimal(11), BigDecimal.valueOf(f1)); + Assertions.assertEquals(res.getBigDecimal(12), BigDecimal.valueOf(d)); + + } + + @Test + public void testException() { + ObjectMapper objectMapper = Utils.getObjectMapper(); + + ArrayNode rows = objectMapper.createArrayNode(); + ObjectNode row = objectMapper.createObjectNode(); + + ArrayNode exceptionNode = objectMapper.createArrayNode(); + exceptionNode.add("Test exception"); + row.putIfAbsent("Exceptions", exceptionNode); + + rows.add(row); + + String columns = "[ { \"ColumnName\": \"a\", \"ColumnType\": \"bool\" }, { \"ColumnName\": \"b\", " + + "\"ColumnType\": \"string\" }, { \"ColumnName\": \"c\", \"ColumnType\": \"datetime\" }, { " + + "\"ColumnName\": \"d\", \"ColumnType\": \"decimal\" }, { \"ColumnName\": \"e\", " + + "\"ColumnType\": \"dynamic\" }, { \"ColumnName\": \"f\", \"ColumnType\": \"guid\" }, { " + + "\"ColumnName\": \"g\", \"ColumnType\": \"int\" }, { \"ColumnName\": \"h\", \"ColumnType\": " + + "\"long\" }, { \"ColumnName\": \"i\", \"ColumnType\": \"real\" }, { \"ColumnName\": \"j\", " + + "\"ColumnType\": \"timespan\" },{ \"ColumnName\": \"k\", \"ColumnType\": \"short\" } ]"; + + Assertions.assertThrows(KustoServiceQueryError.class, () -> { + new KustoResultSetTable(objectMapper.readTree("{\"TableName\":\"Table_0\"," + + "\"Columns\":" + columns + ",\"Rows\":" + + rows + "}")); + }); + } } diff --git a/ingest/pom.xml b/ingest/pom.xml index 6b5b0f03..0ddeb8e1 100644 --- a/ingest/pom.xml +++ b/ingest/pom.xml @@ -229,11 +229,6 @@ ${junit.version} test - - org.json - json - ${json.version} - org.jetbrains annotations @@ -272,4 +267,4 @@ ${io.vavr.version} - \ No newline at end of file + diff --git a/ingest/src/main/java/com/microsoft/azure/kusto/ingest/IngestionProperties.java b/ingest/src/main/java/com/microsoft/azure/kusto/ingest/IngestionProperties.java index 9f647073..9d80b8e3 100644 --- a/ingest/src/main/java/com/microsoft/azure/kusto/ingest/IngestionProperties.java +++ b/ingest/src/main/java/com/microsoft/azure/kusto/ingest/IngestionProperties.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.azure.kusto.data.Ensure; +import com.microsoft.azure.kusto.data.Utils; import com.microsoft.azure.kusto.ingest.exceptions.IngestionClientException; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.TextStringBuilder; @@ -228,13 +229,13 @@ Map getIngestionProperties() throws IOException { } } - ObjectMapper objectMapper = new ObjectMapper(); + ObjectMapper objectMapper = Utils.getObjectMapper(); String tagsAsJson = objectMapper.writeValueAsString(tags); fullAdditionalProperties.put("tags", tagsAsJson); } if (!ingestIfNotExists.isEmpty()) { - ObjectMapper objectMapper = new ObjectMapper(); + ObjectMapper objectMapper = Utils.getObjectMapper(); String ingestIfNotExistsJson = objectMapper.writeValueAsString(ingestIfNotExists); fullAdditionalProperties.put("ingestIfNotExists", ingestIfNotExistsJson); } @@ -247,7 +248,7 @@ Map getIngestionProperties() throws IOException { fullAdditionalProperties.put("ingestionMappingReference", mappingReference); fullAdditionalProperties.put("ingestionMappingType", ingestionMapping.getIngestionMappingKind().getKustoValue()); } else if (ingestionMapping.getColumnMappings() != null) { - ObjectMapper objectMapper = new ObjectMapper(); + ObjectMapper objectMapper = Utils.getObjectMapper(); objectMapper.setVisibility(PropertyAccessor.ALL, Visibility.NONE); objectMapper.setVisibility(PropertyAccessor.FIELD, Visibility.ANY); diff --git a/ingest/src/main/java/com/microsoft/azure/kusto/ingest/ManagedStreamingIngestClient.java b/ingest/src/main/java/com/microsoft/azure/kusto/ingest/ManagedStreamingIngestClient.java index 4d0c2ffd..59041ddf 100644 --- a/ingest/src/main/java/com/microsoft/azure/kusto/ingest/ManagedStreamingIngestClient.java +++ b/ingest/src/main/java/com/microsoft/azure/kusto/ingest/ManagedStreamingIngestClient.java @@ -17,7 +17,6 @@ import org.apache.http.client.utils.URIBuilder; import org.jetbrains.annotations.Nullable; -import org.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -281,13 +280,9 @@ public IngestionResult ingestFromStream(StreamSourceInfo streamSourceInfo, Inges && e.getCause().getCause() != null && e.getCause().getCause() instanceof DataWebException) { DataWebException webException = (DataWebException) e.getCause().getCause(); - try { - OneApiError oneApiError = webException.getApiError(); - if (oneApiError.isPermanent()) { - throw e; - } - } catch (JSONException je) { - log.info("Failed to parse json in exception, continuing.", je); + OneApiError oneApiError = webException.getApiError(); + if (oneApiError.isPermanent()) { + throw e; } } diff --git a/ingest/src/main/java/com/microsoft/azure/kusto/ingest/QueuedIngestClientImpl.java b/ingest/src/main/java/com/microsoft/azure/kusto/ingest/QueuedIngestClientImpl.java index b84a1dfe..9ad86ff7 100644 --- a/ingest/src/main/java/com/microsoft/azure/kusto/ingest/QueuedIngestClientImpl.java +++ b/ingest/src/main/java/com/microsoft/azure/kusto/ingest/QueuedIngestClientImpl.java @@ -4,10 +4,7 @@ package com.microsoft.azure.kusto.ingest; import com.fasterxml.jackson.databind.ObjectMapper; -import com.microsoft.azure.kusto.data.Client; -import com.microsoft.azure.kusto.data.ClientFactory; -import com.microsoft.azure.kusto.data.Ensure; -import com.microsoft.azure.kusto.data.HttpClientProperties; +import com.microsoft.azure.kusto.data.*; import com.microsoft.azure.kusto.data.auth.ConnectionStringBuilder; import com.microsoft.azure.kusto.ingest.exceptions.IngestionClientException; import com.microsoft.azure.kusto.ingest.exceptions.IngestionServiceException; @@ -134,7 +131,7 @@ public IngestionResult ingestFromBlob(BlobSourceInfo blobSourceInfo, IngestionPr tableStatuses.add(ingestionBlobInfo.getIngestionStatusInTable()); } - ObjectMapper objectMapper = new ObjectMapper(); + ObjectMapper objectMapper = Utils.getObjectMapper(); String serializedIngestionBlobInfo = objectMapper.writeValueAsString(ingestionBlobInfo); azureStorageClient.postMessageToQueue( diff --git a/ingest/src/test/java/com/microsoft/azure/kusto/ingest/E2ETest.java b/ingest/src/test/java/com/microsoft/azure/kusto/ingest/E2ETest.java index 71b147cc..4670ac7f 100644 --- a/ingest/src/test/java/com/microsoft/azure/kusto/ingest/E2ETest.java +++ b/ingest/src/test/java/com/microsoft/azure/kusto/ingest/E2ETest.java @@ -3,10 +3,10 @@ package com.microsoft.azure.kusto.ingest; -import com.microsoft.azure.kusto.data.ClientImpl; -import com.microsoft.azure.kusto.data.ClientRequestProperties; -import com.microsoft.azure.kusto.data.KustoOperationResult; -import com.microsoft.azure.kusto.data.KustoResultSetTable; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.microsoft.azure.kusto.data.*; import com.microsoft.azure.kusto.data.auth.CloudInfo; import com.microsoft.azure.kusto.data.auth.ConnectionStringBuilder; import com.microsoft.azure.kusto.data.exceptions.DataClientException; @@ -23,8 +23,6 @@ import org.apache.commons.lang3.time.StopWatch; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; -import org.json.JSONArray; -import org.json.JSONObject; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assumptions; @@ -78,6 +76,7 @@ class E2ETest { private static String tableName; private static final String mappingReference = "mappingRef"; private static final String tableColumns = "(rownumber:int, rowguid:string, xdouble:real, xfloat:real, xbool:bool, xint16:int, xint32:int, xint64:long, xuint8:long, xuint16:long, xuint32:long, xuint64:long, xdate:datetime, xsmalltext:string, xtext:string, xnumberAsText:string, xtime:timespan, xtextWithNulls:string, xdynamicWithNulls:dynamic)"; + private ObjectMapper objectMapper = Utils.getObjectMapper(); @BeforeAll public static void setUp() throws IOException { @@ -226,16 +225,18 @@ private void assertRowCount(int expectedRowsCount, boolean checkViaJson) { if (checkViaJson) { String result = queryClient.executeToJsonResult(databaseName, String.format("%s | count", tableName)); - JSONArray jsonArray = new JSONArray(result); - JSONObject primaryResult = null; - for (Object o : jsonArray) { - if (o.toString() != null && o.toString().matches(".*\"TableKind\"\\s*:\\s*\"PrimaryResult\".*")) { - primaryResult = new JSONObject(o.toString()); + JsonNode jsonNode = objectMapper.readTree(result); + ArrayNode jsonArray = jsonNode.isArray() ? (ArrayNode) jsonNode : null; + JsonNode primaryResult = null; + Assertions.assertNotNull(jsonArray, "JsonArray cant be null since we need to retrieve primary result values out of it. "); + for (JsonNode o : jsonArray) { + if (o != null && o.toString().matches(".*\"TableKind\"\\s*:\\s*\"PrimaryResult\".*")) { + primaryResult = objectMapper.readTree(o.toString()); break; } } - assertNotNull(primaryResult); - actualRowsCount = (Integer) ((JSONArray) ((JSONArray) primaryResult.get("Rows")).get(0)).get(0) - currentCount; + Assertions.assertNotNull(primaryResult, "Primary result cant be null since we need the row count"); + actualRowsCount = (primaryResult.get("Rows")).get(0).get(0).asInt() - currentCount; } else { KustoOperationResult result = queryClient.execute(databaseName, String.format("%s | count", tableName)); KustoResultSetTable mainTableResult = result.getPrimaryResults(); diff --git a/ingest/src/test/java/com/microsoft/azure/kusto/ingest/ResourceManagerTest.java b/ingest/src/test/java/com/microsoft/azure/kusto/ingest/ResourceManagerTest.java index b7574a34..0991048b 100644 --- a/ingest/src/test/java/com/microsoft/azure/kusto/ingest/ResourceManagerTest.java +++ b/ingest/src/test/java/com/microsoft/azure/kusto/ingest/ResourceManagerTest.java @@ -6,12 +6,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.azure.kusto.data.Client; import com.microsoft.azure.kusto.data.KustoOperationResult; +import com.microsoft.azure.kusto.data.Utils; import com.microsoft.azure.kusto.data.exceptions.DataClientException; import com.microsoft.azure.kusto.data.exceptions.DataServiceException; import com.microsoft.azure.kusto.data.exceptions.KustoServiceQueryError; import com.microsoft.azure.kusto.ingest.exceptions.IngestionClientException; import com.microsoft.azure.kusto.ingest.exceptions.IngestionServiceException; -import org.json.JSONException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -21,8 +21,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; @@ -41,7 +39,7 @@ class ResourceManagerTest { private static final String SUCCESS_QUEUE = "successQueue"; @BeforeAll - static void setUp() throws DataClientException, DataServiceException, JSONException, KustoServiceQueryError, IOException { + static void setUp() throws DataClientException, DataServiceException, KustoServiceQueryError, IOException { when(clientMock.execute(Commands.INGESTION_RESOURCES_SHOW_COMMAND)) .thenReturn(generateIngestionResourcesResult()); @@ -119,7 +117,8 @@ void GetIngestionResource_SuccessfulIngestionQueue_ReturnCorrectQueue() resourceManager.getIngestionResource(ResourceManager.ResourceType.SUCCESSFUL_INGESTIONS_QUEUE)); } - static KustoOperationResult generateIngestionResourcesResult() throws JSONException, KustoServiceQueryError, IOException { + static KustoOperationResult generateIngestionResourcesResult() throws KustoServiceQueryError, IOException { + ObjectMapper objectMapper = Utils.getObjectMapper(); List> valuesList = new ArrayList<>(); valuesList.add(new ArrayList<>((Arrays.asList("SecuredReadyForAggregationQueue", QUEUE_1)))); valuesList.add(new ArrayList<>((Arrays.asList("SecuredReadyForAggregationQueue", QUEUE_2)))); @@ -128,7 +127,7 @@ static KustoOperationResult generateIngestionResourcesResult() throws JSONExcept valuesList.add(new ArrayList<>((Arrays.asList("TempStorage", STORAGE_1)))); valuesList.add(new ArrayList<>((Arrays.asList("TempStorage", STORAGE_2)))); valuesList.add(new ArrayList<>((Arrays.asList("IngestionsStatusTable", STATUS_TABLE)))); - String listAsJson = new ObjectMapper().writeValueAsString(valuesList); + String listAsJson = objectMapper.writeValueAsString(valuesList); String response = "{\"Tables\":[{\"TableName\":\"Table_0\",\"Columns\":[{\"ColumnName\":\"ResourceTypeName\"," + "\"DataType\":\"String\",\"ColumnType\":\"string\"},{\"ColumnName\":\"StorageRoot\",\"DataType\":" + "\"String\",\"ColumnType\":\"string\"}],\"Rows\":" @@ -137,10 +136,11 @@ static KustoOperationResult generateIngestionResourcesResult() throws JSONExcept return new KustoOperationResult(response, "v1"); } - static KustoOperationResult generateIngestionAuthTokenResult() throws JSONException, KustoServiceQueryError, IOException { + static KustoOperationResult generateIngestionAuthTokenResult() throws KustoServiceQueryError, IOException { + ObjectMapper objectMapper = Utils.getObjectMapper(); List> valuesList = new ArrayList<>(); valuesList.add(new ArrayList<>((Collections.singletonList(AUTH_TOKEN)))); - String listAsJson = new ObjectMapper().writeValueAsString(valuesList); + String listAsJson = objectMapper.writeValueAsString(valuesList); String response = "{\"Tables\":[{\"TableName\":\"Table_0\",\"Columns\":[{\"ColumnName\":\"AuthorizationContext\",\"DataType\":\"String\",\"ColumnType\":\"string\"}],\"Rows\":" + diff --git a/pom.xml b/pom.xml index 30f81b66..11232b2d 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ - 3.2.0 + 3.3.0 UTF-8 1.8 @@ -44,8 +44,7 @@ 1.11.0 4.5.13 4.4.15 - 20201115 - 2.12.5 + 2.13.3 8.6.6 1.3.7 1.21.0 diff --git a/quickstart/pom.xml b/quickstart/pom.xml index b3e57312..3662a2f2 100644 --- a/quickstart/pom.xml +++ b/quickstart/pom.xml @@ -33,7 +33,7 @@ - 3.2.0 + 3.3.0 1.8 3.2.0 3.8.1 @@ -125,10 +125,5 @@ slf4j-simple ${slf4j.version} - - org.json - json - ${json.version} - diff --git a/quickstart/src/main/java/com/microsoft/azure/kusto/quickstart/SampleApp.java b/quickstart/src/main/java/com/microsoft/azure/kusto/quickstart/SampleApp.java index 85095bac..95fe1ff7 100644 --- a/quickstart/src/main/java/com/microsoft/azure/kusto/quickstart/SampleApp.java +++ b/quickstart/src/main/java/com/microsoft/azure/kusto/quickstart/SampleApp.java @@ -308,7 +308,7 @@ public static void main(String[] args) { private static ConfigJson loadConfigs() { File configFile = new File(".\\" + SampleApp.configFileName); try { - ObjectMapper mapper = new ObjectMapper(); + ObjectMapper mapper = com.microsoft.azure.kusto.data.Utils.getObjectMapper(); mapper.configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); return mapper.readValue(configFile, ConfigJson.class); diff --git a/samples/src/main/java/TableStatus.java b/samples/src/main/java/TableStatus.java index 63d0fa43..8cecb9cd 100644 --- a/samples/src/main/java/TableStatus.java +++ b/samples/src/main/java/TableStatus.java @@ -2,6 +2,7 @@ // Licensed under the MIT License. import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.azure.kusto.data.Utils; import com.microsoft.azure.kusto.data.auth.ConnectionStringBuilder; import com.microsoft.azure.kusto.ingest.IngestClient; import com.microsoft.azure.kusto.ingest.IngestClientFactory; @@ -44,7 +45,7 @@ public static void main(String[] args) { statuses = ingestionResult.getIngestionStatusCollection(); } - ObjectMapper objectMapper = new ObjectMapper(); + ObjectMapper objectMapper = Utils.getObjectMapper(); String resultAsJson = objectMapper.writeValueAsString(statuses.get(0)); System.out.println(resultAsJson); } catch (Exception e) {