From a8ae9e0f90f2e88878580722cf025e37f8255b00 Mon Sep 17 00:00:00 2001 From: Chen Dai <46505291+dai-chen@users.noreply.github.com> Date: Tue, 4 Aug 2020 15:30:15 -0700 Subject: [PATCH] Merge #584, #600, #609 for OD 1.8 (#658) * Issue 584 Fix object/nested field select issue * Issue 600 Fix CAST bool to integer issue * Issue 609 Support queries end with semi colon --- src/main/antlr/OpenDistroSqlParser.g4 | 2 +- .../sql/esdomain/mapping/FieldMapping.java | 27 ++-- .../executor/format/DescribeResultSet.java | 2 +- .../sql/executor/format/Schema.java | 2 + .../sql/executor/format/SelectResultSet.java | 4 +- .../sql/query/ESActionFactory.java | 5 +- .../sql/utils/SQLFunctions.java | 13 +- .../sql/antlr/SyntaxAnalysisTest.java | 7 +- .../esdomain/mapping/FieldMappingTest.java | 51 ++++++- .../sql/esintgtest/ObjectFieldSelectIT.java | 127 ++++++++++++++++++ .../sql/esintgtest/QueryIT.java | 5 + .../sql/esintgtest/SQLFunctionsIT.java | 49 +++++++ .../sql/esintgtest/SQLIntegTestCase.java | 7 +- .../sql/esintgtest/TestUtils.java | 39 ++++++ .../sql/esintgtest/TestsConstants.java | 1 + .../sql/unittest/utils/SQLFunctionsTest.java | 27 ++-- .../sql/util/MatcherUtils.java | 30 ++++- .../resources/deep_nested_index_data.json | 2 + 18 files changed, 359 insertions(+), 41 deletions(-) create mode 100644 src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ObjectFieldSelectIT.java create mode 100644 src/test/resources/deep_nested_index_data.json diff --git a/src/main/antlr/OpenDistroSqlParser.g4 b/src/main/antlr/OpenDistroSqlParser.g4 index ce30892e16..86c5c89d20 100644 --- a/src/main/antlr/OpenDistroSqlParser.g4 +++ b/src/main/antlr/OpenDistroSqlParser.g4 @@ -32,7 +32,7 @@ options { tokenVocab=OpenDistroSqlLexer; } // Root rule root - : sqlStatement? EOF + : sqlStatement? SEMI? EOF ; // Only SELECT, DELETE, SHOW and DSCRIBE are supported for now diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMapping.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMapping.java index 0bbacc0327..56d830723d 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMapping.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMapping.java @@ -16,6 +16,7 @@ package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping; import com.amazon.opendistroforelasticsearch.sql.domain.Field; +import com.amazon.opendistroforelasticsearch.sql.executor.format.DescribeResultSet; import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; import java.util.Map; @@ -119,24 +120,26 @@ public String path() { } /** - * Used to retrieve the type of fields from metaData map structures for both regular and nested fields + * Find field type in ES Get Field Mapping API response. Note that Get Field Mapping API does NOT return + * the type for object or nested field. In this case, object type is used as default under the assumption + * that the field queried here must exist (which is true if semantic analyzer is enabled). + * + * @return field type if found in mapping, otherwise "object" type returned */ @SuppressWarnings("unchecked") public String type() { - FieldMappingMetaData metaData = typeMappings.get(fieldName); + FieldMappingMetaData metaData = typeMappings.getOrDefault(fieldName, FieldMappingMetaData.NULL); + if (metaData.isNull()) { + return DescribeResultSet.DEFAULT_OBJECT_DATATYPE; + } + Map source = metaData.sourceAsMap(); String[] fieldPath = fieldName.split("\\."); - /* - * When field is not nested the metaData source is fieldName -> type - * When it is nested or contains "." in general (ex. fieldName.nestedName) the source is nestedName -> type - */ - String root = (fieldPath.length == 1) ? fieldName : fieldPath[1]; - Map fieldMapping = (Map) source.get(root); - for (int i = 2; i < fieldPath.length; i++) { - fieldMapping = (Map) fieldMapping.get(fieldPath[i]); - } - + // For object/nested field, fieldName is full path though only innermost field name present in mapping + // For example, fieldName='employee.location.city', metaData='{"city":{"type":"text"}}' + String innermostFieldName = (fieldPath.length == 1) ? fieldName : fieldPath[fieldPath.length - 1]; + Map fieldMapping = (Map) source.get(innermostFieldName); return (String) fieldMapping.get("type"); } diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DescribeResultSet.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DescribeResultSet.java index 04044f8360..1ef3cebfb0 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DescribeResultSet.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/DescribeResultSet.java @@ -40,7 +40,7 @@ public class DescribeResultSet extends ResultSet { * You are not required to set the field type to object explicitly, as this is the default value. * https://www.elastic.co/guide/en/elasticsearch/reference/current/object.html */ - private static final String DEFAULT_OBJECT_DATATYPE = "object"; + public static final String DEFAULT_OBJECT_DATATYPE = "object"; private IndexStatement statement; private Object queryResult; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/Schema.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/Schema.java index 552f1e172b..2fb379bd8c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/Schema.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/Schema.java @@ -114,6 +114,8 @@ public enum Type { DATE, // Date types BOOLEAN, // Boolean types BINARY, // Binary types + OBJECT, + NESTED, INTEGER_RANGE, FLOAT_RANGE, LONG_RANGE, DOUBLE_RANGE, DATE_RANGE; // Range types public String nameLowerCase() { diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java index ece165a951..b596ea2388 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/format/SelectResultSet.java @@ -437,6 +437,7 @@ private List populateColumns(Query query, String[] fieldNames, Ma // _score is a special case since it is not included in typeMappings, so it is checked for here if (fieldName.equals(SCORE)) { columns.add(new Schema.Column(fieldName, fetchAlias(fieldName, fieldMap), Schema.Type.FLOAT)); + continue; } /* * Methods are also a special case as their type cannot be determined from typeMappings, so it is checked @@ -465,6 +466,7 @@ private List populateColumns(Query query, String[] fieldNames, Ma fetchMethodReturnType(fieldIndex, methodField) ) ); + continue; } /* @@ -473,7 +475,7 @@ private List populateColumns(Query query, String[] fieldNames, Ma * explicitly selected. */ FieldMapping field = new FieldMapping(fieldName, typeMappings, fieldMap); - if (typeMappings.containsKey(fieldName) && !field.isMetaField()) { + if (!field.isMetaField()) { if (field.isMultiField() && !field.isSpecified()) { continue; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/ESActionFactory.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/ESActionFactory.java index 4251362022..f54636d90a 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/ESActionFactory.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/query/ESActionFactory.java @@ -87,8 +87,11 @@ public static QueryAction create(Client client, String sql) public static QueryAction create(Client client, QueryActionRequest request) throws SqlParseException, SQLFeatureNotSupportedException { String sql = request.getSql(); - // Linebreak matcher + // Remove line breaker anywhere and semicolon at the end sql = sql.replaceAll("\\R", " ").trim(); + if (sql.endsWith(";")) { + sql = sql.substring(0, sql.length() - 1); + } switch (getFirstWord(sql)) { case "SELECT": diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java index 3d08c0e162..5916d8007c 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java @@ -973,13 +973,10 @@ public String getCastScriptStatement(String name, String castType, List String castFieldName = String.format("doc['%s'].value", paramers.get(0).toString()); switch (StringUtils.toUpper(castType)) { case "INT": - return String.format("def %s = Double.parseDouble(%s.toString()).intValue()", name, castFieldName); case "LONG": - return String.format("def %s = Double.parseDouble(%s.toString()).longValue()", name, castFieldName); case "FLOAT": - return String.format("def %s = Double.parseDouble(%s.toString()).floatValue()", name, castFieldName); case "DOUBLE": - return String.format("def %s = Double.parseDouble(%s.toString()).doubleValue()", name, castFieldName); + return getCastToNumericValueScript(name, castFieldName, StringUtils.toLower(castType)); case "STRING": return String.format("def %s = %s.toString()", name, castFieldName); case "DATETIME": @@ -990,6 +987,14 @@ public String getCastScriptStatement(String name, String castType, List } } + private String getCastToNumericValueScript(String varName, String docValue, String targetType) { + String script = + "def %1$s = (%2$s instanceof boolean) " + + "? (%2$s ? 1 : 0) " + + ": Double.parseDouble(%2$s.toString()).%3$sValue()"; + return StringUtils.format(script, varName, docValue, targetType); + } + /** * Returns return type of script function. This is simple approach, that might be not the best solution in the long * term. For example - for JDBC, if the column type in index is INTEGER, and the query is "select column+5", current diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/SyntaxAnalysisTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/SyntaxAnalysisTest.java index e5d93a3504..ee409efe38 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/SyntaxAnalysisTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/antlr/SyntaxAnalysisTest.java @@ -79,7 +79,7 @@ public void missingWhereKeywordShouldThrowException() { expectValidationFailWithErrorMessage( "SELECT * FROM accounts age = 1", "offending symbol [=]", // parser thought 'age' is alias of 'accounts' and failed at '=' - "Expecting", "'WHERE'" // "Expecting tokens in {, 'INNER', 'JOIN', ... 'WHERE', ','}" + "Expecting", ";" // "Expecting tokens in {, ';'}" ); } @@ -130,6 +130,11 @@ public void arithmeticExpressionInWhereClauseShouldPass() { validate("SELECT * FROM accounts WHERE age + 1 = 10"); } + @Test + public void queryEndWithSemiColonShouldPass() { + validate("SELECT * FROM accounts;"); + } + private void expectValidationFailWithErrorMessage(String query, String... messages) { exception.expect(SyntaxAnalysisException.class); exception.expectMessage(allOf(Arrays.stream(messages). diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappingTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappingTest.java index a83c0bc0e4..d8f27c2dab 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappingTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappingTest.java @@ -15,18 +15,20 @@ package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping; +import static java.util.Collections.emptyMap; +import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + import com.amazon.opendistroforelasticsearch.sql.domain.Field; import com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils; -import org.hamcrest.Matcher; -import org.junit.Test; - +import com.google.common.collect.ImmutableMap; import java.util.Arrays; import java.util.Map; import java.util.stream.Collectors; - -import static java.util.Collections.emptyMap; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; +import org.elasticsearch.common.bytes.BytesArray; +import org.hamcrest.Matcher; +import org.junit.Test; /** * Unit test for {@code FieldMapping} with trivial methods ignored such as isSpecified, isMetaField etc. @@ -81,6 +83,35 @@ public void testMultiFieldIsNotProperty() { ); } + @Test + public void testUnknownFieldTreatedAsObject() { + assertThat( + new FieldMapping("employee"), + hasType("object") + ); + } + + @Test + public void testDeepNestedField() { + assertThat( + new FieldMapping( + "employee.location.city", + ImmutableMap.of( + "employee.location.city", + new FieldMappingMetaData("employee.location.city", new BytesArray( + "{\n" + + " \"city\" : {\n" + + " \"type\" : \"text\"\n" + + " }\n" + + "}") + ) + ), + emptyMap() + ), + hasType("text") + ); + } + private Matcher isWildcardSpecified(boolean isMatched) { return MatcherUtils.featureValueOf("is field match wildcard specified in query", is(isMatched), @@ -93,6 +124,12 @@ private Matcher isPropertyField(boolean isProperty) { FieldMapping::isPropertyField); } + private Matcher hasType(String expected) { + return MatcherUtils.featureValueOf("type", + is(expected), + FieldMapping::type); + } + private Map fieldsSpecifiedInQuery(String...fieldNames) { return Arrays.stream(fieldNames). collect(Collectors.toMap(name -> name, diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ObjectFieldSelectIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ObjectFieldSelectIT.java new file mode 100644 index 0000000000..898cc6dba2 --- /dev/null +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/ObjectFieldSelectIT.java @@ -0,0 +1,127 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.esintgtest; + +import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestsConstants.TEST_INDEX_DEEP_NESTED; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRows; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema; + +import com.amazon.opendistroforelasticsearch.sql.utils.StringUtils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.Test; + +/** + * Integration test for Elasticsearch object field (and nested field). + * This class is focused on simple SELECT-FROM query to ensure right column + * number and value is returned. + */ +public class ObjectFieldSelectIT extends SQLIntegTestCase { + + @Override + protected void init() throws Exception { + loadIndex(Index.DEEP_NESTED); + } + + @Test + public void testSelectObjectFieldItself() { + JSONObject response = new JSONObject(query("SELECT city FROM %s")); + + verifySchema(response, schema("city", null, "object")); + + // Expect object field itself is returned in a single cell + verifyDataRows(response, + rows(new JSONObject( + "{\n" + + " \"name\": \"Seattle\",\n" + + " \"location\": {\"latitude\": 10.5}\n" + + "}") + ) + ); + } + + @Test + public void testSelectObjectInnerFields() { + JSONObject response = new JSONObject(query( + "SELECT city.location, city.location.latitude FROM %s")); + + verifySchema(response, + schema("city.location", null, "object"), + schema("city.location.latitude", null, "double") + ); + + // Expect inner regular or object field returned in its single cell + verifyDataRows(response, + rows( + new JSONObject("{\"latitude\": 10.5}"), + 10.5 + ) + ); + } + + @Test + public void testSelectNestedFieldItself() { + JSONObject response = new JSONObject(query("SELECT projects FROM %s")); + + // Nested field is absent in ES Get Field Mapping response either hence "object" used + verifySchema(response, schema("projects", null, "object")); + + // Expect nested field itself is returned in a single cell + verifyDataRows(response, + rows(new JSONArray( + "[\n" + + " {\"name\": \"AWS Redshift Spectrum querying\"},\n" + + " {\"name\": \"AWS Redshift security\"},\n" + + " {\"name\": \"AWS Aurora security\"}\n" + + "]") + ) + ); + } + + @Test + public void testSelectObjectFieldOfArrayValuesItself() { + JSONObject response = new JSONObject(query("SELECT accounts FROM %s")); + + // Expect the entire list of values is returned just like a nested field + verifyDataRows(response, + rows(new JSONArray( + "[\n" + + " {\"id\": 1},\n" + + " {\"id\": 2}\n" + + "]") + ) + ); + } + + @Test + public void testSelectObjectFieldOfArrayValuesInnerFields() { + JSONObject response = new JSONObject(query("SELECT accounts.id FROM %s")); + + // We don't support flatten object field of list value so expect null returned + verifyDataRows(response, rows(JSONObject.NULL)); + } + + private String query(String sql) { + return executeQuery( + StringUtils.format(sql, TEST_INDEX_DEEP_NESTED), + "jdbc" + ); + } + +} diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/QueryIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/QueryIT.java index b3d7adfd87..dd6cee92b5 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/QueryIT.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/QueryIT.java @@ -90,6 +90,11 @@ protected void init() throws Exception { loadIndex(Index.BANK_WITH_NULL_VALUES); } + @Test + public void queryEndWithSemiColonTest() { + executeQuery(StringUtils.format("SELECT * FROM %s;", TEST_INDEX_BANK), "jdbc"); + } + @Test public void searchTypeTest() throws IOException { JSONObject response = executeQuery(String.format(Locale.ROOT, "SELECT * FROM %s LIMIT 1000", diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java index c8019c55c2..73e67dc39d 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java @@ -75,6 +75,7 @@ public class SQLFunctionsIT extends SQLIntegTestCase { @Override protected void init() throws Exception { loadIndex(Index.ACCOUNT); + loadIndex(Index.BANK); loadIndex(Index.ONLINE); loadIndex(Index.DATE); } @@ -371,6 +372,54 @@ public void castFieldToDatetimeWithGroupByJdbcFormatTest() { rows("2019-09-25T02:04:13.469Z")); } + @Test + public void castBoolFieldToNumericValueInSelectClause() { + JSONObject response = + executeJdbcRequest( + "SELECT " + + " male, " + + " CAST(male AS INT) AS cast_int, " + + " CAST(male AS LONG) AS cast_long, " + + " CAST(male AS FLOAT) AS cast_float, " + + " CAST(male AS DOUBLE) AS cast_double " + + "FROM " + TestsConstants.TEST_INDEX_BANK + " " + + "WHERE account_number = 1 OR account_number = 13" + ); + + verifySchema(response, + schema("male", "boolean"), + schema("cast_int", "integer"), + schema("cast_long", "long"), + schema("cast_float", "float"), + schema("cast_double", "double") + ); + verifyDataRows(response, + rows(true, 1, 1, 1, 1), + rows(false, 0, 0, 0, 0) + ); + } + + @Test + public void castBoolFieldToNumericValueWithGroupByAlias() { + JSONObject response = + executeJdbcRequest( + "SELECT " + + "CAST(male AS INT) AS cast_int, " + + "COUNT(*) " + + "FROM " + TestsConstants.TEST_INDEX_BANK + " " + + "GROUP BY cast_int" + ); + + verifySchema(response, + schema("cast_int", "cast_int", "double"), //Type is double due to query plan fail to infer + schema("COUNT(*)", "integer") + ); + verifyDataRows(response, + rows("0", 3), + rows("1", 4) + ); + } + @Test public void castStatementInWhereClauseGreaterThanTest() { JSONObject response = executeJdbcRequest("SELECT balance FROM " + TEST_INDEX_ACCOUNT diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java index 30c3e709f2..826534cc7f 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLIntegTestCase.java @@ -45,6 +45,7 @@ import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getBankWithNullValuesIndexMapping; import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getDateIndexMapping; import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getDateTimeIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getDeepNestedIndexMapping; import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getNestedSimpleIndexMapping; import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getDogIndexMapping; import static com.amazon.opendistroforelasticsearch.sql.esintgtest.TestUtils.getDogs2IndexMapping; @@ -512,7 +513,11 @@ public enum Index { NESTED_SIMPLE(TestsConstants.TEST_INDEX_NESTED_SIMPLE, "_doc", getNestedSimpleIndexMapping(), - "src/test/resources/nested_simple.json"); + "src/test/resources/nested_simple.json"), + DEEP_NESTED(TestsConstants.TEST_INDEX_DEEP_NESTED, + "_doc", + getDeepNestedIndexMapping(), + "src/test/resources/deep_nested_index_data.json"); private final String name; private final String type; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java index 2a495c622a..9679caa315 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestUtils.java @@ -688,6 +688,45 @@ public static String getNestedSimpleIndexMapping() { " }" + "}"; } + + public static String getDeepNestedIndexMapping() { + return "{\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"city\": {\n" + + " \"properties\": {\n" + + " \"name\": {\n" + + " \"type\": \"keyword\"\n" + + " },\n" + + " \"location\": {\n" + + " \"properties\": {\n" + + " \"latitude\": {\n" + + " \"type\": \"double\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"account\": {\n" + + " \"properties\": {\n" + + " \"id\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"projects\": {\n" + + " \"type\": \"nested\",\n" + + " \"properties\": {\n" + + " \"name\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "} "; + } + public static void loadBulk(Client client, String jsonPath, String defaultIndex) throws Exception { System.out.println(String.format("Loading file %s into elasticsearch cluster", jsonPath)); String absJsonPath = getResourceFilePath(jsonPath); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestsConstants.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestsConstants.java index 66c01ad244..ab8790e0f0 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestsConstants.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/TestsConstants.java @@ -48,6 +48,7 @@ public class TestsConstants { public final static String TEST_INDEX_WEBLOG = TEST_INDEX + "_weblog"; public final static String TEST_INDEX_DATE = TEST_INDEX + "_date"; public final static String TEST_INDEX_DATE_TIME = TEST_INDEX + "_datetime"; + public final static String TEST_INDEX_DEEP_NESTED = TEST_INDEX + "_deep_nested"; public final static String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/SQLFunctionsTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/SQLFunctionsTest.java index 6d48f5c004..5617a478e8 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/SQLFunctionsTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/utils/SQLFunctionsTest.java @@ -15,6 +15,9 @@ package com.amazon.opendistroforelasticsearch.sql.unittest.utils; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + import com.alibaba.druid.sql.ast.SQLDataType; import com.alibaba.druid.sql.ast.SQLDataTypeImpl; import com.alibaba.druid.sql.ast.expr.SQLCastExpr; @@ -28,25 +31,20 @@ import com.amazon.opendistroforelasticsearch.sql.domain.ScriptMethodField; import com.amazon.opendistroforelasticsearch.sql.exception.SqlParseException; import com.amazon.opendistroforelasticsearch.sql.executor.format.Schema; -import com.amazon.opendistroforelasticsearch.sql.parser.FieldMaker; import com.amazon.opendistroforelasticsearch.sql.utils.SQLFunctions; import com.google.common.collect.ImmutableList; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import org.elasticsearch.common.collect.Tuple; import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertTrue; - public class SQLFunctionsTest { - private SQLFunctions sqlFunctions; + private SQLFunctions sqlFunctions = new SQLFunctions(); @Rule public ExpectedException exceptionRule = ExpectedException.none(); @@ -99,4 +97,15 @@ public void testCastReturnType() { final Schema.Type returnType = sqlFunctions.getScriptFunctionReturnType(field, resolvedType); Assert.assertEquals(returnType, Schema.Type.INTEGER); } + + @Test + public void testCastIntStatementScript() throws SqlParseException { + assertEquals( + "def result = (doc['age'].value instanceof boolean) " + + "? (doc['age'].value ? 1 : 0) " + + ": Double.parseDouble(doc['age'].value.toString()).intValue()", + sqlFunctions.getCastScriptStatement( + "result", "int", Arrays.asList(new KVValue("age"))) + ); + } } \ No newline at end of file diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MatcherUtils.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MatcherUtils.java index cdf81928cb..a05804be07 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MatcherUtils.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/util/MatcherUtils.java @@ -172,6 +172,11 @@ public static void verifySome(JSONArray array, Matcher... matchers) { } } + public static TypeSafeMatcher schema(String expectedName, + String expectedType) { + return schema(expectedName, null, expectedType); + } + public static TypeSafeMatcher schema(String expectedName, String expectedAlias, String expectedType) { return new TypeSafeMatcher() { @Override @@ -202,9 +207,28 @@ public void describeTo(Description description) { @Override protected boolean matchesSafely(JSONArray array) { - List actualObjects = new ArrayList<>(); - array.iterator().forEachRemaining(actualObjects::add); - return Arrays.asList(expectedObjects).equals(actualObjects); + if (array.length() != expectedObjects.length) { + return false; + } + + for (int i = 0; i < expectedObjects.length; i++) { + Object expected = expectedObjects[i]; + boolean isEqual; + + // Use similar() because JSONObject/JSONArray.equals() only check if same reference + if (expected instanceof JSONObject) { + isEqual = ((JSONObject) expected).similar(array.get(i)); + } else if (expected instanceof JSONArray) { + isEqual = ((JSONArray) expected).similar(array.get(i)); + } else { + isEqual = expected.equals(array.get(i)); + } + + if (!isEqual) { + return false; + } + } + return true; } }; } diff --git a/src/test/resources/deep_nested_index_data.json b/src/test/resources/deep_nested_index_data.json new file mode 100644 index 0000000000..5e3883d516 --- /dev/null +++ b/src/test/resources/deep_nested_index_data.json @@ -0,0 +1,2 @@ +{"index":{"_id":"1"}} +{"city": {"name": "Seattle", "location": {"latitude": 10.5}}, "accounts": [{"id": 1}, {"id": 2}], "projects": [{"name": "AWS Redshift Spectrum querying"}, {"name": "AWS Redshift security"}, {"name": "AWS Aurora security"}] }