From 919f423c2904dff9d3df5c743aec0b0338f03a80 Mon Sep 17 00:00:00 2001 From: Ioana Tagirta Date: Wed, 9 Oct 2024 17:06:04 +0200 Subject: [PATCH 01/10] fix tests --- .../xpack/esql/core/type/DataType.java | 3 +++ .../function/scalar/convert/ToBoolean.java | 4 +++- .../function/scalar/convert/ToCartesianPoint.java | 2 ++ .../function/scalar/convert/ToCartesianShape.java | 2 ++ .../function/scalar/convert/ToDatetime.java | 4 +++- .../function/scalar/convert/ToDouble.java | 4 +++- .../function/scalar/convert/ToGeoPoint.java | 2 ++ .../function/scalar/convert/ToGeoShape.java | 2 ++ .../expression/function/scalar/convert/ToIP.java | 4 +++- .../function/scalar/convert/ToInteger.java | 5 ++++- .../expression/function/scalar/convert/ToLong.java | 4 +++- .../function/scalar/convert/ToString.java | 4 +++- .../function/scalar/convert/ToUnsignedLong.java | 4 +++- .../function/scalar/convert/ToVersion.java | 4 +++- .../predicate/operator/comparison/Equals.java | 3 ++- .../predicate/operator/comparison/GreaterThan.java | 3 ++- .../operator/comparison/GreaterThanOrEqual.java | 3 ++- .../predicate/operator/comparison/LessThan.java | 3 ++- .../operator/comparison/LessThanOrEqual.java | 3 ++- .../predicate/operator/comparison/NotEquals.java | 3 ++- .../xpack/esql/type/EsqlDataTypeConverter.java | 4 ++++ .../xpack/esql/analysis/VerifierTests.java | 6 +++--- .../function/AbstractFunctionTestCase.java | 14 ++++++++++++-- .../function/scalar/string/ConcatTests.java | 2 +- .../predicate/operator/comparison/EqualsTests.java | 4 ++-- .../comparison/GreaterThanOrEqualTests.java | 2 +- .../operator/comparison/GreaterThanTests.java | 2 +- .../operator/comparison/LessThanOrEqualTests.java | 2 +- .../operator/comparison/LessThanTests.java | 2 +- 29 files changed, 77 insertions(+), 27 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 5041c96128a1e..4b95ed7af53fc 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -370,6 +370,9 @@ public static boolean isUnsupported(DataType from) { } public static boolean isString(DataType t) { + if (EsqlCorePlugin.SEMANTIC_TEXT_FEATURE_FLAG.isEnabled() && t == SEMANTIC_TEXT) { + return true; + } return t == KEYWORD || t == TEXT; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java index 06cc993456433..58f706f01c49a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java @@ -28,6 +28,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToBoolean; @@ -47,7 +48,8 @@ public class ToBoolean extends AbstractConvertFunction { Map.entry(DOUBLE, ToBooleanFromDoubleEvaluator.Factory::new), Map.entry(LONG, ToBooleanFromLongEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToBooleanFromUnsignedLongEvaluator.Factory::new), - Map.entry(INTEGER, ToBooleanFromIntEvaluator.Factory::new) + Map.entry(INTEGER, ToBooleanFromIntEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToBooleanFromStringEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java index 60a25fc91d50d..db3617fe589eb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java @@ -25,6 +25,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial; @@ -38,6 +39,7 @@ public class ToCartesianPoint extends AbstractConvertFunction { private static final Map EVALUATORS = Map.ofEntries( Map.entry(CARTESIAN_POINT, (fieldEval, source) -> fieldEval), Map.entry(KEYWORD, ToCartesianPointFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToCartesianPointFromStringEvaluator.Factory::new), Map.entry(TEXT, ToCartesianPointFromStringEvaluator.Factory::new) ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java index 03ac4bdf48243..5ec5d2a4ac741 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java @@ -26,6 +26,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT; import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial; @@ -40,6 +41,7 @@ public class ToCartesianShape extends AbstractConvertFunction { Map.entry(CARTESIAN_POINT, (fieldEval, source) -> fieldEval), Map.entry(CARTESIAN_SHAPE, (fieldEval, source) -> fieldEval), Map.entry(KEYWORD, ToCartesianShapeFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToCartesianShapeFromStringEvaluator.Factory::new), Map.entry(TEXT, ToCartesianShapeFromStringEvaluator.Factory::new) ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java index c66ba7f87a1c5..17dcde81ae330 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java @@ -30,6 +30,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToLong; @@ -49,7 +50,8 @@ public class ToDatetime extends AbstractConvertFunction { Map.entry(TEXT, ToDatetimeFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToLongFromDoubleEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToLongFromUnsignedLongEvaluator.Factory::new), - Map.entry(INTEGER, ToLongFromIntEvaluator.Factory::new) // CastIntToLongEvaluator would be a candidate, but not MV'd + Map.entry(INTEGER, ToLongFromIntEvaluator.Factory::new), // CastIntToLongEvaluator would be a candidate, but not MV'd + Map.entry(SEMANTIC_TEXT, ToDatetimeFromStringEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java index de88281e7dbd1..c0b671d32bb38 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java @@ -30,6 +30,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToDouble; @@ -49,7 +50,8 @@ public class ToDouble extends AbstractConvertFunction { Map.entry(INTEGER, ToDoubleFromIntEvaluator.Factory::new), // CastIntToDoubleEvaluator would be a candidate, but not MV'd Map.entry(DataType.COUNTER_DOUBLE, (field, source) -> field), Map.entry(DataType.COUNTER_INTEGER, ToDoubleFromIntEvaluator.Factory::new), - Map.entry(DataType.COUNTER_LONG, ToDoubleFromLongEvaluator.Factory::new) + Map.entry(DataType.COUNTER_LONG, ToDoubleFromLongEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToDoubleFromStringEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java index 51cb08137a58c..2dc5f1662b786 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java @@ -25,6 +25,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial; @@ -38,6 +39,7 @@ public class ToGeoPoint extends AbstractConvertFunction { private static final Map EVALUATORS = Map.ofEntries( Map.entry(GEO_POINT, (fieldEval, source) -> fieldEval), Map.entry(KEYWORD, ToGeoPointFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToGeoPointFromStringEvaluator.Factory::new), Map.entry(TEXT, ToGeoPointFromStringEvaluator.Factory::new) ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java index 00e9fb3e598f1..3d6e3f3baa348 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java @@ -26,6 +26,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToSpatial; @@ -40,6 +41,7 @@ public class ToGeoShape extends AbstractConvertFunction { Map.entry(GEO_POINT, (fieldEval, source) -> fieldEval), Map.entry(GEO_SHAPE, (fieldEval, source) -> fieldEval), Map.entry(KEYWORD, ToGeoShapeFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToGeoShapeFromStringEvaluator.Factory::new), Map.entry(TEXT, ToGeoShapeFromStringEvaluator.Factory::new) ); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java index 6df85948d94ef..cd161744bfc86 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java @@ -25,6 +25,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.IP; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToIP; @@ -34,7 +35,8 @@ public class ToIP extends AbstractConvertFunction { private static final Map EVALUATORS = Map.ofEntries( Map.entry(IP, (field, source) -> field), Map.entry(KEYWORD, ToIPFromStringEvaluator.Factory::new), - Map.entry(TEXT, ToIPFromStringEvaluator.Factory::new) + Map.entry(TEXT, ToIPFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToIPFromStringEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java index 1785160594a78..254cf5fe536de 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java @@ -25,11 +25,13 @@ import java.util.Map; import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; +import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.safeToInt; @@ -52,7 +54,8 @@ public class ToInteger extends AbstractConvertFunction { Map.entry(DOUBLE, ToIntegerFromDoubleEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToIntegerFromUnsignedLongEvaluator.Factory::new), Map.entry(LONG, ToIntegerFromLongEvaluator.Factory::new), - Map.entry(DataType.COUNTER_INTEGER, (fieldEval, source) -> fieldEval) + Map.entry(COUNTER_INTEGER, (fieldEval, source) -> fieldEval), + Map.entry(SEMANTIC_TEXT, ToIntegerFromStringEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java index e5f138df159cd..1a4026e3a5cce 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java @@ -31,6 +31,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.safeDoubleToLong; @@ -51,7 +52,8 @@ public class ToLong extends AbstractConvertFunction { Map.entry(UNSIGNED_LONG, ToLongFromUnsignedLongEvaluator.Factory::new), Map.entry(INTEGER, ToLongFromIntEvaluator.Factory::new), // CastIntToLongEvaluator would be a candidate, but not MV'd Map.entry(DataType.COUNTER_LONG, (field, source) -> field), - Map.entry(DataType.COUNTER_INTEGER, ToLongFromIntEvaluator.Factory::new) + Map.entry(DataType.COUNTER_INTEGER, ToLongFromIntEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToLongFromStringEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java index f9bc15c4d6903..5bf572d3bee3f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java @@ -36,6 +36,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.IP; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; @@ -65,7 +66,8 @@ public class ToString extends AbstractConvertFunction implements EvaluatorMapper Map.entry(GEO_POINT, ToStringFromGeoPointEvaluator.Factory::new), Map.entry(CARTESIAN_POINT, ToStringFromCartesianPointEvaluator.Factory::new), Map.entry(CARTESIAN_SHAPE, ToStringFromCartesianShapeEvaluator.Factory::new), - Map.entry(GEO_SHAPE, ToStringFromGeoShapeEvaluator.Factory::new) + Map.entry(GEO_SHAPE, ToStringFromGeoShapeEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, (fieldEval, source) -> fieldEval) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java index bfbfcf44b3945..82df3a0589255 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java @@ -30,6 +30,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.booleanToUnsignedLong; @@ -53,7 +54,8 @@ public class ToUnsignedLong extends AbstractConvertFunction { Map.entry(TEXT, ToUnsignedLongFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToUnsignedLongFromDoubleEvaluator.Factory::new), Map.entry(LONG, ToUnsignedLongFromLongEvaluator.Factory::new), - Map.entry(INTEGER, ToUnsignedLongFromIntEvaluator.Factory::new) + Map.entry(INTEGER, ToUnsignedLongFromIntEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToUnsignedLongFromStringEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java index f6002c3c6bb17..296ddb35c3c41 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersion.java @@ -24,6 +24,7 @@ import java.util.Map; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToVersion; @@ -38,7 +39,8 @@ public class ToVersion extends AbstractConvertFunction { private static final Map EVALUATORS = Map.ofEntries( Map.entry(VERSION, (fieldEval, source) -> fieldEval), Map.entry(KEYWORD, ToVersionFromStringEvaluator.Factory::new), - Map.entry(TEXT, ToVersionFromStringEvaluator.Factory::new) + Map.entry(TEXT, ToVersionFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToVersionFromStringEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java index 614d9aa3ec920..730653e53dc14 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java @@ -43,7 +43,8 @@ public class Equals extends EsqlBinaryComparison implements Negatable, DataType> entry : signatures().entrySet()) { List types = entry.getKey(); + DataType returnType = entry.getValue(); for (int i = 0; i < args.size() && i < types.size(); i++) { typesFromSignature.get(i).add(types.get(i).esNameIfPossible()); } - returnFromSignature.add(entry.getValue().esNameIfPossible()); + if (DataType.UNDER_CONSTRUCTION.containsKey(returnType) == false) { + returnFromSignature.add(returnType.esNameIfPossible()); + } } for (int i = 0; i < args.size(); i++) { @@ -746,7 +749,14 @@ public static void testFunctionInfo() { ); } - Set returnTypes = Arrays.stream(description.returnType()).collect(Collectors.toCollection(TreeSet::new)); + Set underConstructionTypes = DataType.UNDER_CONSTRUCTION.keySet() + .stream() + .map(dataType -> dataType.esNameIfPossible()) + .collect(Collectors.toCollection(TreeSet::new)); + + Set returnTypes = Arrays.stream(description.returnType()) + .filter(typeName -> underConstructionTypes.contains(typeName) == false) + .collect(Collectors.toCollection(TreeSet::new)); assertEquals(returnFromSignature, returnTypes); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java index 2ad953c9296b7..35b2fd276e92a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java @@ -48,7 +48,7 @@ public static Iterable parameters() { for (int length = 4; length < 100; length++) { suppliers(suppliers, length); } - Set supported = Set.of(DataType.NULL, DataType.KEYWORD, DataType.TEXT); + Set supported = Set.of(DataType.NULL, DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT); List> supportedPerPosition = List.of(supported, supported); for (DataType lhs : DataType.types()) { if (lhs == DataType.NULL || DataType.isRepresentable(lhs) == false) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java index 66edd56e32f20..0fb416584b472 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java @@ -217,8 +217,8 @@ public static Iterable parameters() { } private static String typeErrorString = - "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long, text, " - + "unsigned_long or version"; + "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long," + + " semantic_text, text, unsigned_long or version"; @Override protected Expression build(Source source, List args) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java index a4d1bf69796e0..a4f1a19e135ef 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java @@ -150,7 +150,7 @@ public static Iterable parameters() { o, v, t, - (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, text, unsigned_long or version" + (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" ) ) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java index d3fede5c2e2ce..86a4676e35009 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java @@ -150,7 +150,7 @@ public static Iterable parameters() { o, v, t, - (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, text, unsigned_long or version" + (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" ) ) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java index 3b8270c1576fd..5793f26ecd447 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java @@ -150,7 +150,7 @@ public static Iterable parameters() { o, v, t, - (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, text, unsigned_long or version" + (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" ) ) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java index 647988fe35326..e8f9f26a76f43 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java @@ -150,7 +150,7 @@ public static Iterable parameters() { o, v, t, - (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, text, unsigned_long or version" + (l, p) -> "date_nanos, datetime, double, integer, ip, keyword, long, semantic_text, text, unsigned_long or version" ) ) ); From 7b56752dabeb46f80331c14dc37d3f14cb295cfb Mon Sep 17 00:00:00 2001 From: Ioana Tagirta Date: Thu, 10 Oct 2024 15:45:55 +0200 Subject: [PATCH 02/10] Add CSV tests --- .../src/main/resources/semantic_text.csv-spec | 874 ++++++++++++++++++ .../xpack/esql/analysis/Verifier.java | 4 + 2 files changed, 878 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec index 683bcdc3f7490..409f275d0baf1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec @@ -173,3 +173,877 @@ host:keyword | semantic_text_field:semantic_text "host2" | all we have to decide is what to do with the time that is given to us "host3" | be excellent to each other ; + +convertToBool +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_bool(st_bool) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:bool +1 | false +2 | true +3 | null +; + +convertToCartesianPoint +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_cartesianpoint(st_cartesian_point) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:cartesian_point +1 | "POINT(4297.11 -1475.53)" +2 | "POINT(7580.93 2272.77)" +3 | null +; + +convertToCartesianShape +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_cartesianshape(st_cartesian_shape) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:cartesian_shape +1 | null +2 | null +3 | null +; + +convertToDatetime +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_datetime(st_datetime) +| KEEP _id, result +| SORT _id, result +; + +_id:keyword|result:datetime +1 | 1953-09-02T00:00:00.000Z +2 | 2023-09-24T15:57:00.000Z +3 | null +; + +convertToDouble +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_double(st_double) +| KEEP _id, result +| SORT _id +; + +_id:keyword|result:double +1 | 5.20128E11 +2 | 4541.11 +3 | null +; + +convertToGeopoint +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_geopoint(st_geopoint) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:geo_point +1 | "POINT(42.97109630194 14.7552534413725)" +2 | "POINT(37.97109630194 21.7552534413725)" +3 | null +; + +convertToGeoshape +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_geoshape(st_geoshape) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:geo_shape +1 | "POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))" +2 | "POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))" +3 | null +; + +convertToInteger +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_integer(st_integer) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:integer +1 | 23 +2 | 122 +3 | null +; + +convertToIp +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_ip(st_ip) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:ip +1 | 1.1.1.1 +2 | 1.1.2.1 +3 | null +; + +convertToLong +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_long(st_long) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:long +1 | 2147483648 +2 | 123 +3 | null +; + +convertToUnsignedLong +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_unsigned_long(st_unsigned_long) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:unsigned_long +1 | 2147483648 +2 | 2147483648.2 +3 | null +; + +convertToVersion +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_version(st_version) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:version +1 | 1.2.3 +2 | 9.0.0 +3 | null +; + +concat +required_capability: semantic_text_type + +FROM semantic_text +| EVAL result = concat("", semantic_text_field, "") +| KEEP result +| SORT result +; + +result:keyword +all we have to decide is what to do with the time that is given to us +be excellent to each other +live long and prosper +; + + +endsWith +required_capability: semantic_text_type + +FROM semantic_text +| WHERE ends_with(semantic_text_field, "er") +| KEEP semantic_text_field +| SORT semantic_text_field +; + +semantic_text_field:semantic_text +be excellent to each other +live long and prosper +; + +left +required_capability: semantic_text_type + +FROM semantic_text +| EVAL result = left(semantic_text_field, 2) +| SORT result +| KEEP result +; + +result:keyword +al +be +li +; + +length +required_capability: semantic_text_type + +FROM semantic_text +| EVAL result = length(st_version) +| KEEP result +| SORT result +; + +result:integer +5 +5 +null +; + +locate +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = locate(semantic_text_field, "all") +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:integer +1 | 0 +2 | 1 +3 | 0 +; + +ltrim +required_capability: semantic_text_type + +FROM semantic_text +| EVAL result = ltrim(semantic_text_field) +| SORT result +| KEEP result +; + +result:semantic_text +all we have to decide is what to do with the time that is given to us +be excellent to each other +live long and prosper +; + +repeat +required_capability: semantic_text_type + +FROM semantic_text +| EVAL result = repeat(semantic_text_field, 2) +| WHERE length(semantic_text_field) < 25 +| KEEP result +; + +result:keyword +live long and prosperlive long and prosper +; + +replace +required_capability: semantic_text_type + +FROM semantic_text +| EVAL result = replace(semantic_text_field, "excellent", "good") +| WHERE length(semantic_text_field) < 30 +| KEEP result +| SORT result +; + +result:keyword +be good to each other +live long and prosper +; + +right +required_capability: semantic_text_type + +FROM semantic_text +| EVAL result = right(semantic_text_field, 2) +| KEEP result +| SORT result +; + +result:keyword +er +er +us +; + +rtrim +required_capability: semantic_text_type + +FROM semantic_text +| EVAL result = rtrim(semantic_text_field) +| KEEP result +| SORT result +; + +result:semantic_text +all we have to decide is what to do with the time that is given to us +be excellent to each other +live long and prosper +; + +split +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = split(st_version, ".") +| SORT _id +| KEEP result +; + +result:keyword +["1", "2", "3"] +["9", "0", "0"] +null +; + +startsWith +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = starts_with(semantic_text_field, "be") +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:bool +1 | false +2 | false +3 | true +; + +substring +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = substring(semantic_text_field, 2, 1) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:keyword +1 | i +2 | l +3 | e +; + + +toLower +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_lower(st_cartesian_point) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:semantic_text +1 | point(4297.11 -1475.53) +2 | point(7580.93 2272.77) +3 | null +; + +toUpper +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_upper(semantic_text_field) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:semantic_text +1 | LIVE LONG AND PROSPER +2 | ALL WE HAVE TO DECIDE IS WHAT TO DO WITH THE TIME THAT IS GIVEN TO US +3 | BE EXCELLENT TO EACH OTHER +; + +trim +required_capability: semantic_text_type + +FROM semantic_text +| EVAL result = trim(semantic_text_field) +| SORT result +| KEEP result +; + +result:semantic_text +all we have to decide is what to do with the time that is given to us +be excellent to each other +live long and prosper +; + +mvAppend +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = mv_append(st_integer, st_long) +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:semantic_text +1 | ["23", "2147483648"] +2 | ["122", "123"] +3 | null +; + +mvConcat +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL x = mv_append(st_integer, st_long) +| EVAL result = mv_concat(x, "; ") +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:keyword +1 | 23; 2147483648 +2 | 122; 123 +3 | null +; + +mvCount +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL x = mv_append(st_integer, st_long) +| EVAL result = mv_count(x) +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:integer +1 | 2 +2 | 2 +3 | null +; + +mvDedupe +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL x = mv_append(st_integer, st_long) +| EVAL x = mv_append(x, st_integer) +| EVAL result = mv_dedupe(x) +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:semantic_text +1 | ["23", "2147483648"] +2 | ["122", "123"] +3 | null +; + +mvFirst +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL x = mv_append(st_integer, st_long) +| EVAL result = mv_first(x) +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:semantic_text +1 | 23 +2 | 122 +3 | null +; + +mvLast +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL x = mv_append(st_integer, st_long) +| EVAL result = mv_last(x) +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:semantic_text +1 | 2147483648 +2 | 123 +3 | null +; + +mvMax +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL x = mv_append(st_integer, st_long) +| EVAL result = mv_max(x) +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:semantic_text +1 | 23 +2 | 123 +3 | null +; + +mvMin +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL x = mv_append(st_integer, st_long) +| EVAL result = mv_min(x) +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:semantic_text +1 | 2147483648 +2 | 122 +3 | null +; + +mvSlice +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL x = mv_append(st_integer, st_long) +| EVAL x = mv_append(x, st_integer) +| EVAL result = mv_slice(x, 1, 2) +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:semantic_text +1 | ["2147483648", "23"] +2 | ["123", "122"] +3 | null +; + +mvSort +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL x = mv_append(st_integer, st_long) +| EVAL result = mv_sort(x, "DESC") +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:semantic_text +1 | ["23", "2147483648"] +2 | ["123", "122"] +3 | null +; + + +mvZip +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL x = mv_append(st_integer, st_long) +| EVAL y = mv_append(st_double, st_integer) +| EVAL result = mv_zip(x, y, " + ") +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:keyword +1 | ["23 + 5.20128E11", "2147483648 + 23"] +2 | ["122 + 4541.11", "123 + 122"] +3 | null +; + +equalityWithConstant +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_ip == "1.1.1.1" +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | true +2 | false +3 | null +; + +equalityBetweenFields +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_long == st_unsigned_long +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | true +2 | false +3 | null +; + + +inequalityWithConstant +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_ip != "1.1.1.1" +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | false +2 | true +3 | null +; + +inequalityBetweenFields +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_long != st_unsigned_long +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | false +2 | true +3 | null +; + +lessThanWithConstant +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = semantic_text_field < "bye!" +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | false +2 | true +3 | true +; + +lessThanBetweenFields +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = semantic_text_field < st_version +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | false +2 | false +3 | null +; + + +lessThanOrEqualToWithConstant +required_capability: semantic_text_type + + +FROM semantic_text METADATA _id +| EVAL result = semantic_text_field <= "be excellent to each other" +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | false +2 | true +3 | true +; + + +lessThanOrEqualToBetweenFields +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_integer <= st_long +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | false +2 | true +3 | null +; + + +greaterThanWithConstant +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = semantic_text_field > "bye!" +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | true +2 | false +3 | false +; + + +greaterThanBetweenFields +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = semantic_text_field > st_version +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | true +2 | true +3 | null +; + +greaterThanOrEqualToWithConstant +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = semantic_text_field >= "be excellent to each other" +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | true +2 | false +3 | true +; + +greaterThanOrEqualToBetweenFields +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_integer >= st_long +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | true +2 | false +3 | null +; + +isNull +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_integer IS NULL +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | false +2 | false +3 | true +; + +isNotNull +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_integer IS NOT NULL +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | true +2 | true +3 | false +; + +cast +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_bool::BOOL +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:bool +1 | false +2 | true +3 | null +; + +in +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_integer IN ("123", "23") +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | true +2 | false +3 | null +; + +like +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = semantic_text_field LIKE "all*" +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | false +2 | true +3 | false +; + +rlike +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = st_version RLIKE "[0-9].[0-9].[0-9]" +| KEEP _id, result +| SORT _id +; + +_id: keyword | result:bool +1 | true +2 | true +3 | null +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index e2717cd9af0d1..aeb86dbd74c96 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.analysis; import org.elasticsearch.common.logging.LoggerMessageFormat; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable; import org.elasticsearch.xpack.esql.core.expression.Alias; @@ -512,6 +513,9 @@ public static Failure validateBinaryComparison(BinaryComparison bc) { allowed.add(DataType.GEO_SHAPE); allowed.add(DataType.CARTESIAN_POINT); allowed.add(DataType.CARTESIAN_SHAPE); + if (EsqlCapabilities.Cap.SEMANTIC_TEXT_TYPE.isEnabled()) { + allowed.add(DataType.SEMANTIC_TEXT); + } if (bc instanceof Equals || bc instanceof NotEquals) { allowed.add(DataType.BOOLEAN); } From fb83c4c62cd73dadc679a5acf1f8618eec5b52a6 Mon Sep 17 00:00:00 2001 From: Ioana Tagirta Date: Tue, 15 Oct 2024 17:01:20 +0200 Subject: [PATCH 03/10] Add function tests --- .../function/scalar/string/Concat.java | 5 +++-- .../operator/comparison/GreaterThan.java | 4 ++-- .../AbstractMultivalueFunctionTestCase.java | 2 +- .../scalar/multivalue/MvAppendTests.java | 16 +++++++++++++++ .../scalar/multivalue/MvSliceTests.java | 17 ++++++++++++++++ .../scalar/multivalue/MvSortTests.java | 14 +++++++++++++ .../function/scalar/string/ConcatTests.java | 3 ++- .../function/scalar/string/LeftTests.java | 14 +++++++++++++ .../function/scalar/string/LengthTests.java | 10 ++++++++++ .../function/scalar/string/RLikeTests.java | 2 +- .../function/scalar/string/RepeatTests.java | 16 +++++++++++++++ .../function/scalar/string/ReverseTests.java | 2 +- .../function/scalar/string/RightTests.java | 13 ++++++++++++ .../function/scalar/string/SplitTests.java | 2 +- .../scalar/string/SubstringTests.java | 19 ++++++++++++++++++ .../function/scalar/string/ToLowerTests.java | 2 ++ .../function/scalar/string/ToUpperTests.java | 2 ++ .../scalar/string/WildcardLikeTests.java | 2 +- .../operator/comparison/InTests.java | 20 +++++++++++++++++-- 19 files changed, 153 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java index 46ecc9e026d3d..87f7339a2a228 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java @@ -50,8 +50,9 @@ public class Concat extends EsqlScalarFunction { ) public Concat( Source source, - @Param(name = "string1", type = { "keyword", "text" }, description = "Strings to concatenate.") Expression first, - @Param(name = "string2", type = { "keyword", "text" }, description = "Strings to concatenate.") List rest + @Param(name = "string1", type = { "keyword", "semantic_text", "text" }, description = "Strings to concatenate.") Expression first, + @Param(name = "string2", type = { "keyword", "semantic_text", "text" }, description = "Strings to concatenate.") List< + ? extends Expression> rest ) { super(source, Stream.concat(Stream.of(first), rest.stream()).toList()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java index 0842e9d5f9e02..8f82836fcd715 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java @@ -53,12 +53,12 @@ public GreaterThan( Source source, @Param( name = "lhs", - type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" }, + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "semantic_text", "text", "unsigned_long", "version" }, description = "An expression." ) Expression left, @Param( name = "rhs", - type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" }, + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "semantic_text", "text", "unsigned_long", "version" }, description = "An expression." ) Expression right ) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java index 13920fac26f5b..2ff4f5dd87b30 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java @@ -123,7 +123,7 @@ protected static void bytesRefs( Function expectedDataType, BiFunction, Matcher> matcher ) { - for (DataType type : new DataType[] { DataType.KEYWORD, DataType.TEXT, DataType.IP, DataType.VERSION }) { + for (DataType type : new DataType[] { DataType.KEYWORD, DataType.TEXT, DataType.IP, DataType.VERSION, DataType.SEMANTIC_TEXT }) { if (type != DataType.IP) { cases.add( new TestCaseSupplier( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java index 37f4464c8b3ca..f603466859b5d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java @@ -266,6 +266,22 @@ private static void bytesRefs(List suppliers) { equalTo(result) ); })); + + suppliers.add(new TestCaseSupplier(List.of(DataType.SEMANTIC_TEXT, DataType.SEMANTIC_TEXT), () -> { + List field1 = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); + List field2 = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.SEMANTIC_TEXT, "field1"), + new TestCaseSupplier.TypedData(field2, DataType.SEMANTIC_TEXT, "field2") + ), + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataType.SEMANTIC_TEXT, + equalTo(result) + ); + })); } private static void nulls(List suppliers) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java index e5bac422805af..4997c296dfa8d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java @@ -339,5 +339,22 @@ private static void bytesRefs(List suppliers) { equalTo(start == end ? field.get(start) : field.subList(start, end + 1)) ); })); + + suppliers.add(new TestCaseSupplier(List.of(DataType.SEMANTIC_TEXT, DataType.INTEGER, DataType.INTEGER), () -> { + List field = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); + int length = field.size(); + int start = randomIntBetween(0, length - 1); + int end = randomIntBetween(start, length - 1); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field, DataType.SEMANTIC_TEXT, "field"), + new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"), + new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end") + ), + "MvSliceBytesRefEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]", + DataType.SEMANTIC_TEXT, + equalTo(start == end ? field.get(start) : field.subList(start, end + 1)) + ); + })); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java index d07ed2aeae887..8d18f0372b645 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java @@ -184,6 +184,20 @@ private static void bytesRefs(List suppliers) { equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList()) ); })); + + suppliers.add(new TestCaseSupplier(List.of(DataType.SEMANTIC_TEXT, DataType.KEYWORD), () -> { + List field = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); + BytesRef order = new BytesRef("ASC"); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field, DataType.SEMANTIC_TEXT, "field"), + new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral() + ), + "MvSortBytesRef[field=Attribute[channel=0], order=true]", + DataType.SEMANTIC_TEXT, + equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList()) + ); + })); } private static void nulls(List suppliers) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java index 35b2fd276e92a..3cf855568ef49 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java @@ -72,6 +72,7 @@ private static void suppliers(List suppliers, int length) { if (length > 3) { suppliers.add(supplier("ascii", DataType.KEYWORD, length, () -> randomAlphaOfLengthBetween(1, 10))); suppliers.add(supplier("unicode", DataType.TEXT, length, () -> randomRealisticUnicodeOfLengthBetween(1, 10))); + suppliers.add(supplier("unicode", DataType.SEMANTIC_TEXT, length, () -> randomRealisticUnicodeOfLengthBetween(1, 10))); } else { add(suppliers, "ascii", length, () -> randomAlphaOfLengthBetween(1, 10)); add(suppliers, "unicode", length, () -> randomRealisticUnicodeOfLengthBetween(1, 10)); @@ -99,7 +100,7 @@ private static TestCaseSupplier supplier(String name, DataType type, int length, private static void add(List suppliers, String name, int length, Supplier valueSupplier) { Map>> permutations = new HashMap>>(); - List supportedDataTypes = List.of(DataType.KEYWORD, DataType.TEXT); + List supportedDataTypes = List.of(DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT); permutations.put(0, List.of(List.of(DataType.KEYWORD), List.of(DataType.TEXT))); for (int v = 0; v < length - 1; v++) { List> current = permutations.get(v); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java index 627c46da025ea..e4e54a9e0935f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LeftTests.java @@ -167,6 +167,20 @@ public static Iterable parameters() { ); })); + suppliers.add(new TestCaseSupplier("semantic_text as input", List.of(DataType.SEMANTIC_TEXT, DataType.INTEGER), () -> { + String text = randomUnicodeOfLengthBetween(1, 64); + int length = between(1, text.length()); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataType.SEMANTIC_TEXT, "str"), + new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length") + ), + "LeftEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataType.KEYWORD, + equalTo(new BytesRef(unicodeLeftSubstring(text, length))) + ); + })); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { case 0 -> "string"; case 1 -> "integer"; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java index 6ae5a9d961398..ba4c8c8ce1ea4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LengthTests.java @@ -73,6 +73,16 @@ private static List makeTestCases(String title, Supplier new TestCaseSupplier.TestCase( + List.of(new TestCaseSupplier.TypedData(new BytesRef(text.get()), DataType.SEMANTIC_TEXT, "f")), + "LengthEvaluator[val=Attribute[channel=0]]", + DataType.INTEGER, + equalTo(expectedLength) + ) ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java index dab2fca212ff4..9a49b63645c89 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java @@ -125,7 +125,7 @@ private static void casesForString( } private static void cases(List cases, String title, Supplier textAndPattern, boolean expected) { - for (DataType type : new DataType[] { DataType.KEYWORD, DataType.TEXT }) { + for (DataType type : new DataType[] { DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT }) { cases.add(new TestCaseSupplier(title + " with " + type.esType(), List.of(type, type, DataType.BOOLEAN), () -> { TextAndPattern v = textAndPattern.get(); return new TestCaseSupplier.TestCase( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java index 4d97a2f629c23..8b4ea066fdccb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java @@ -62,6 +62,22 @@ public static Iterable parameters() { ); })); + cases.add( + new TestCaseSupplier("Repeat basic test with semantic_text input", List.of(DataType.SEMANTIC_TEXT, DataType.INTEGER), () -> { + String text = randomAlphaOfLength(10); + int number = between(0, 10); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataType.SEMANTIC_TEXT, "str"), + new TestCaseSupplier.TypedData(number, DataType.INTEGER, "number") + ), + "RepeatEvaluator[str=Attribute[channel=0], number=Attribute[channel=1]]", + DataType.KEYWORD, + equalTo(new BytesRef(text.repeat(number))) + ); + }) + ); + cases.add(new TestCaseSupplier("Repeat with number zero", List.of(DataType.KEYWORD, DataType.INTEGER), () -> { String text = randomAlphaOfLength(10); int number = 0; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java index 2873f18d53957..b677aa1567db4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java @@ -33,7 +33,7 @@ public ReverseTests(@Name("TestCase") Supplier testCa public static Iterable parameters() { List suppliers = new ArrayList<>(); - for (DataType stringType : new DataType[] { DataType.KEYWORD, DataType.TEXT }) { + for (DataType stringType : new DataType[] { DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT }) { for (var supplier : TestCaseSupplier.stringCases(stringType)) { suppliers.add(makeSupplier(supplier)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java index a1ef77a62b67c..bf93ef42ed6ad 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RightTests.java @@ -166,6 +166,19 @@ public static Iterable parameters() { equalTo(new BytesRef(unicodeRightSubstring(text, length))) ); })); + suppliers.add(new TestCaseSupplier("ascii as semantic_text", List.of(DataType.SEMANTIC_TEXT, DataType.INTEGER), () -> { + String text = randomAlphaOfLengthBetween(1, 64); + int length = between(1, text.length()); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataType.SEMANTIC_TEXT, "str"), + new TestCaseSupplier.TypedData(length, DataType.INTEGER, "length") + ), + "RightEvaluator[str=Attribute[channel=0], length=Attribute[channel=1]]", + DataType.KEYWORD, + equalTo(new BytesRef(unicodeRightSubstring(text, length))) + ); + })); return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> switch (p) { case 0 -> "string"; case 1 -> "integer"; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java index b5560f37914a9..ad233a0080d44 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java @@ -42,7 +42,7 @@ public SplitTests(@Name("TestCase") Supplier testCase @ParametersFactory public static Iterable parameters() { List suppliers = new ArrayList<>(); - List supportedDataTyes = List.of(DataType.KEYWORD, DataType.TEXT); + List supportedDataTyes = List.of(DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT); for (DataType sType : supportedDataTyes) { for (DataType dType : supportedDataTyes) { suppliers.add(new TestCaseSupplier("split test " + sType.toString() + " " + dType.toString(), List.of(sType, dType), () -> { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java index 6b934aae775df..ae8a2a1840dfb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SubstringTests.java @@ -72,6 +72,25 @@ public static Iterable parameters() { ); } ), + new TestCaseSupplier( + "Substring basic test with semantic_text input", + List.of(DataType.SEMANTIC_TEXT, DataType.INTEGER, DataType.INTEGER), + () -> { + int start = between(1, 8); + int length = between(1, 10 - start); + String text = randomAlphaOfLength(10); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef(text), DataType.SEMANTIC_TEXT, "str"), + new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"), + new TestCaseSupplier.TypedData(length, DataType.INTEGER, "end") + ), + "SubstringEvaluator[str=Attribute[channel=0], start=Attribute[channel=1], length=Attribute[channel=2]]", + DataType.KEYWORD, + equalTo(new BytesRef(text.substring(start - 1, start + length - 1))) + ); + } + ), new TestCaseSupplier("Substring empty string", List.of(DataType.TEXT, DataType.INTEGER, DataType.INTEGER), () -> { int start = between(1, 8); int length = between(1, 10 - start); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java index 7af1c180fd7b9..cedf12f520dc5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java @@ -45,6 +45,8 @@ public static Iterable parameters() { suppliers.add(supplier("keyword unicode", DataType.KEYWORD, () -> randomUnicodeOfLengthBetween(1, 10))); suppliers.add(supplier("text ascii", DataType.TEXT, () -> randomAlphaOfLengthBetween(1, 10))); suppliers.add(supplier("text unicode", DataType.TEXT, () -> randomUnicodeOfLengthBetween(1, 10))); + suppliers.add(supplier("semantic_text ascii", DataType.SEMANTIC_TEXT, () -> randomAlphaOfLengthBetween(1, 10))); + suppliers.add(supplier("semantic_text unicode", DataType.SEMANTIC_TEXT, () -> randomUnicodeOfLengthBetween(1, 10))); // add null as parameter return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string"); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java index c8bbe03bde411..4145d2cff0a42 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java @@ -45,6 +45,8 @@ public static Iterable parameters() { suppliers.add(supplier("keyword unicode", DataType.KEYWORD, () -> randomUnicodeOfLengthBetween(1, 10))); suppliers.add(supplier("text ascii", DataType.TEXT, () -> randomAlphaOfLengthBetween(1, 10))); suppliers.add(supplier("text unicode", DataType.TEXT, () -> randomUnicodeOfLengthBetween(1, 10))); + suppliers.add(supplier("semantic_text ascii", DataType.SEMANTIC_TEXT, () -> randomAlphaOfLengthBetween(1, 10))); + suppliers.add(supplier("semantic_text unicode", DataType.SEMANTIC_TEXT, () -> randomUnicodeOfLengthBetween(1, 10))); // add null as parameter return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string"); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java index 53b3a99f97b9c..eed2c7379e9e1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java @@ -53,7 +53,7 @@ public static Iterable parameters() { } private static void addCases(List suppliers) { - for (DataType type : new DataType[] { DataType.KEYWORD, DataType.TEXT }) { + for (DataType type : new DataType[] { DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT }) { suppliers.add(new TestCaseSupplier(" with " + type.esType(), List.of(type, type), () -> { BytesRef str = new BytesRef(randomAlphaOfLength(5)); String patternString = randomAlphaOfLength(2); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java index 2a1dfb098a3a4..41d1d3840eb98 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java @@ -187,8 +187,24 @@ private static void bytesRefs(List suppliers, int items) { ); })); - for (DataType type1 : new DataType[] { DataType.KEYWORD, DataType.TEXT }) { - for (DataType type2 : new DataType[] { DataType.KEYWORD, DataType.TEXT }) { + suppliers.add(new TestCaseSupplier("semantic_text", List.of(DataType.SEMANTIC_TEXT, DataType.SEMANTIC_TEXT), () -> { + List inlist = randomList(items, items, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); + Object field = inlist.get(0); + List args = new ArrayList<>(inlist.size() + 1); + for (Object i : inlist) { + args.add(new TestCaseSupplier.TypedData(i, DataType.SEMANTIC_TEXT, "inlist" + i)); + } + args.add(new TestCaseSupplier.TypedData(field, DataType.SEMANTIC_TEXT, "field")); + return new TestCaseSupplier.TestCase( + args, + matchesPattern("InBytesRefEvaluator.*"), + DataType.BOOLEAN, + equalTo(inlist.contains(field)) + ); + })); + + for (DataType type1 : new DataType[] { DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT }) { + for (DataType type2 : new DataType[] { DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT }) { if (type1 == type2 || items > 1) continue; suppliers.add(new TestCaseSupplier(type1 + " " + type2, List.of(type1, type2), () -> { List inlist = randomList(items, items, () -> randomLiteral(type1).value()); From 66e338d60a8d19d4b8e33bc7c0001f45eedc1c01 Mon Sep 17 00:00:00 2001 From: Ioana Tagirta Date: Mon, 21 Oct 2024 17:35:32 +0200 Subject: [PATCH 04/10] Refactor tests --- .../src/main/resources/semantic_text.csv-spec | 203 ++++++++---------- .../function/scalar/string/Concat.java | 4 +- .../operator/comparison/GreaterThan.java | 4 +- 3 files changed, 97 insertions(+), 114 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec index 409f275d0baf1..dfeb3f1f44dac 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec @@ -369,7 +369,6 @@ result:keyword live long and prosper ; - endsWith required_capability: semantic_text_type @@ -547,7 +546,6 @@ _id:keyword | result:keyword 3 | e ; - toLower required_capability: semantic_text_type @@ -597,45 +595,43 @@ mvAppend required_capability: semantic_text_type FROM semantic_text METADATA _id -| EVAL result = mv_append(st_integer, st_long) +| EVAL result = mv_append(st_multi_value, st_long) | KEEP _id, result | SORT _id ; _id: keyword | result:semantic_text -1 | ["23", "2147483648"] -2 | ["122", "123"] -3 | null +1 | ["Hello there!", "This is a random value", "for testing purposes", "2147483648"] +2 | ["nice to meet you", "bye bye!", "123"] +3 | null ; mvConcat required_capability: semantic_text_type FROM semantic_text METADATA _id -| EVAL x = mv_append(st_integer, st_long) -| EVAL result = mv_concat(x, "; ") +| EVAL result = mv_concat(st_multi_value, "; ") | KEEP _id, result | SORT _id ; _id: keyword | result:keyword -1 | 23; 2147483648 -2 | 122; 123 -3 | null +1 | Hello there!; This is a random value; for testing purposes +2 | nice to meet you; bye bye! +3 | null ; mvCount required_capability: semantic_text_type FROM semantic_text METADATA _id -| EVAL x = mv_append(st_integer, st_long) -| EVAL result = mv_count(x) +| EVAL result = mv_count(st_multi_value) | KEEP _id, result | SORT _id ; _id: keyword | result:integer -1 | 2 +1 | 3 2 | 2 3 | null ; @@ -644,132 +640,120 @@ mvDedupe required_capability: semantic_text_type FROM semantic_text METADATA _id -| EVAL x = mv_append(st_integer, st_long) -| EVAL x = mv_append(x, st_integer) -| EVAL result = mv_dedupe(x) +| EVAL result = mv_dedupe(st_multi_value) | KEEP _id, result | SORT _id ; _id: keyword | result:semantic_text -1 | ["23", "2147483648"] -2 | ["122", "123"] -3 | null +1 | ["Hello there!", "This is a random value", "for testing purposes"] +2 | ["nice to meet you", "bye bye!"] +3 | null ; mvFirst required_capability: semantic_text_type FROM semantic_text METADATA _id -| EVAL x = mv_append(st_integer, st_long) -| EVAL result = mv_first(x) +| EVAL result = mv_first(st_multi_value) | KEEP _id, result | SORT _id ; _id: keyword | result:semantic_text -1 | 23 -2 | 122 -3 | null +1 | Hello there! +2 | nice to meet you +3 | null ; mvLast required_capability: semantic_text_type FROM semantic_text METADATA _id -| EVAL x = mv_append(st_integer, st_long) -| EVAL result = mv_last(x) +| EVAL result = mv_last(st_multi_value) | KEEP _id, result | SORT _id ; _id: keyword | result:semantic_text -1 | 2147483648 -2 | 123 -3 | null +1 | for testing purposes +2 | bye bye! +3 | null ; mvMax required_capability: semantic_text_type FROM semantic_text METADATA _id -| EVAL x = mv_append(st_integer, st_long) -| EVAL result = mv_max(x) +| EVAL result = mv_max(st_multi_value) | KEEP _id, result | SORT _id ; _id: keyword | result:semantic_text -1 | 23 -2 | 123 -3 | null +1 | for testing purposes +2 | nice to meet you +3 | null ; mvMin required_capability: semantic_text_type FROM semantic_text METADATA _id -| EVAL x = mv_append(st_integer, st_long) -| EVAL result = mv_min(x) +| EVAL result = mv_min(st_multi_value) | KEEP _id, result | SORT _id ; _id: keyword | result:semantic_text -1 | 2147483648 -2 | 122 -3 | null +1 | Hello there! +2 | bye bye! +3 | null ; mvSlice required_capability: semantic_text_type FROM semantic_text METADATA _id -| EVAL x = mv_append(st_integer, st_long) -| EVAL x = mv_append(x, st_integer) -| EVAL result = mv_slice(x, 1, 2) +| EVAL result = mv_slice(st_multi_value, 1, 2) | KEEP _id, result | SORT _id ; _id: keyword | result:semantic_text -1 | ["2147483648", "23"] -2 | ["123", "122"] -3 | null +1 | ["This is a random value", "for testing purposes"] +2 | bye bye! +3 | null ; mvSort required_capability: semantic_text_type FROM semantic_text METADATA _id -| EVAL x = mv_append(st_integer, st_long) -| EVAL result = mv_sort(x, "DESC") +| EVAL result = mv_sort(st_multi_value, "ASC") | KEEP _id, result | SORT _id ; _id: keyword | result:semantic_text -1 | ["23", "2147483648"] -2 | ["123", "122"] -3 | null +1 | ["Hello there!", "This is a random value", "for testing purposes"] +2 | ["bye bye!", "nice to meet you"] +3 | null ; - mvZip required_capability: semantic_text_type FROM semantic_text METADATA _id -| EVAL x = mv_append(st_integer, st_long) -| EVAL y = mv_append(st_double, st_integer) -| EVAL result = mv_zip(x, y, " + ") +| EVAL result = mv_zip(st_multi_value, st_multi_value, " + ") | KEEP _id, result | SORT _id ; _id: keyword | result:keyword -1 | ["23 + 5.20128E11", "2147483648 + 23"] -2 | ["122 + 4541.11", "123 + 122"] -3 | null +1 | ["Hello there! + Hello there!", "This is a random value + This is a random value", "for testing purposes + for testing purposes"] +2 | ["nice to meet you + nice to meet you", "bye bye! + bye bye!"] +3 | null ; equalityWithConstant @@ -782,9 +766,9 @@ FROM semantic_text METADATA _id ; _id: keyword | result:bool -1 | true -2 | false -3 | null +1 | true +2 | false +3 | null ; equalityBetweenFields @@ -797,12 +781,11 @@ FROM semantic_text METADATA _id ; _id: keyword | result:bool -1 | true -2 | false -3 | null +1 | true +2 | false +3 | null ; - inequalityWithConstant required_capability: semantic_text_type @@ -813,9 +796,9 @@ FROM semantic_text METADATA _id ; _id: keyword | result:bool -1 | false -2 | true -3 | null +1 | false +2 | true +3 | null ; inequalityBetweenFields @@ -828,9 +811,9 @@ FROM semantic_text METADATA _id ; _id: keyword | result:bool -1 | false -2 | true -3 | null +1 | false +2 | true +3 | null ; lessThanWithConstant @@ -843,9 +826,9 @@ FROM semantic_text METADATA _id ; _id: keyword | result:bool -1 | false -2 | true -3 | true +1 | false +2 | true +3 | true ; lessThanBetweenFields @@ -858,9 +841,9 @@ FROM semantic_text METADATA _id ; _id: keyword | result:bool -1 | false -2 | false -3 | null +1 | false +2 | false +3 | null ; @@ -875,9 +858,9 @@ FROM semantic_text METADATA _id ; _id: keyword | result:bool -1 | false -2 | true -3 | true +1 | false +2 | true +3 | true ; @@ -891,9 +874,9 @@ FROM semantic_text METADATA _id ; _id: keyword | result:bool -1 | false -2 | true -3 | null +1 | false +2 | true +3 | null ; @@ -907,9 +890,9 @@ FROM semantic_text METADATA _id ; _id: keyword | result:bool -1 | true -2 | false -3 | false +1 | true +2 | false +3 | false ; @@ -923,9 +906,9 @@ FROM semantic_text METADATA _id ; _id: keyword | result:bool -1 | true -2 | true -3 | null +1 | true +2 | true +3 | null ; greaterThanOrEqualToWithConstant @@ -938,9 +921,9 @@ FROM semantic_text METADATA _id ; _id: keyword | result:bool -1 | true -2 | false -3 | true +1 | true +2 | false +3 | true ; greaterThanOrEqualToBetweenFields @@ -953,9 +936,9 @@ FROM semantic_text METADATA _id ; _id: keyword | result:bool -1 | true -2 | false -3 | null +1 | true +2 | false +3 | null ; isNull @@ -968,9 +951,9 @@ FROM semantic_text METADATA _id ; _id: keyword | result:bool -1 | false -2 | false -3 | true +1 | false +2 | false +3 | true ; isNotNull @@ -983,9 +966,9 @@ FROM semantic_text METADATA _id ; _id: keyword | result:bool -1 | true -2 | true -3 | false +1 | true +2 | true +3 | false ; cast @@ -1013,9 +996,9 @@ FROM semantic_text METADATA _id ; _id: keyword | result:bool -1 | true -2 | false -3 | null +1 | true +2 | false +3 | null ; like @@ -1028,9 +1011,9 @@ FROM semantic_text METADATA _id ; _id: keyword | result:bool -1 | false -2 | true -3 | false +1 | false +2 | true +3 | false ; rlike @@ -1043,7 +1026,7 @@ FROM semantic_text METADATA _id ; _id: keyword | result:bool -1 | true -2 | true -3 | null +1 | true +2 | true +3 | null ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java index 87f7339a2a228..cab56c9bbff54 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java @@ -50,8 +50,8 @@ public class Concat extends EsqlScalarFunction { ) public Concat( Source source, - @Param(name = "string1", type = { "keyword", "semantic_text", "text" }, description = "Strings to concatenate.") Expression first, - @Param(name = "string2", type = { "keyword", "semantic_text", "text" }, description = "Strings to concatenate.") List< + @Param(name = "string1", type = { "keyword", "text" }, description = "Strings to concatenate.") Expression first, + @Param(name = "string2", type = { "keyword", "text" }, description = "Strings to concatenate.") List< ? extends Expression> rest ) { super(source, Stream.concat(Stream.of(first), rest.stream()).toList()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java index 8f82836fcd715..0842e9d5f9e02 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java @@ -53,12 +53,12 @@ public GreaterThan( Source source, @Param( name = "lhs", - type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "semantic_text", "text", "unsigned_long", "version" }, + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" }, description = "An expression." ) Expression left, @Param( name = "rhs", - type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "semantic_text", "text", "unsigned_long", "version" }, + type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" }, description = "An expression." ) Expression right ) { From 0349a645f8a35f587068fd01bbde03c1ec9131ef Mon Sep 17 00:00:00 2001 From: Ioana Tagirta Date: Mon, 21 Oct 2024 18:05:54 +0200 Subject: [PATCH 05/10] spotless --- .../xpack/esql/expression/function/scalar/string/Concat.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java index cab56c9bbff54..46ecc9e026d3d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java @@ -51,8 +51,7 @@ public class Concat extends EsqlScalarFunction { public Concat( Source source, @Param(name = "string1", type = { "keyword", "text" }, description = "Strings to concatenate.") Expression first, - @Param(name = "string2", type = { "keyword", "text" }, description = "Strings to concatenate.") List< - ? extends Expression> rest + @Param(name = "string2", type = { "keyword", "text" }, description = "Strings to concatenate.") List rest ) { super(source, Stream.concat(Stream.of(first), rest.stream()).toList()); } From d33ad0fdd016cf8ba3514990719d9a9a8a85cc3e Mon Sep 17 00:00:00 2001 From: Ioana Tagirta Date: Tue, 22 Oct 2024 11:48:48 +0200 Subject: [PATCH 06/10] Use DataType.stringTypes() where possible --- .../esql/expression/function/scalar/string/RLikeTests.java | 2 +- .../esql/expression/function/scalar/string/ReverseTests.java | 2 +- .../esql/expression/function/scalar/string/SplitTests.java | 5 ++--- .../expression/predicate/operator/comparison/InTests.java | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java index 9a49b63645c89..4f8adf3abaae6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java @@ -125,7 +125,7 @@ private static void casesForString( } private static void cases(List cases, String title, Supplier textAndPattern, boolean expected) { - for (DataType type : new DataType[] { DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT }) { + for (DataType type : DataType.stringTypes()) { cases.add(new TestCaseSupplier(title + " with " + type.esType(), List.of(type, type, DataType.BOOLEAN), () -> { TextAndPattern v = textAndPattern.get(); return new TestCaseSupplier.TestCase( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java index b677aa1567db4..58d52cc02b548 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java @@ -33,7 +33,7 @@ public ReverseTests(@Name("TestCase") Supplier testCa public static Iterable parameters() { List suppliers = new ArrayList<>(); - for (DataType stringType : new DataType[] { DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT }) { + for (DataType stringType : DataType.stringTypes()) { for (var supplier : TestCaseSupplier.stringCases(stringType)) { suppliers.add(makeSupplier(supplier)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java index ad233a0080d44..098be8e1fda37 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/SplitTests.java @@ -42,9 +42,8 @@ public SplitTests(@Name("TestCase") Supplier testCase @ParametersFactory public static Iterable parameters() { List suppliers = new ArrayList<>(); - List supportedDataTyes = List.of(DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT); - for (DataType sType : supportedDataTyes) { - for (DataType dType : supportedDataTyes) { + for (DataType sType : DataType.stringTypes()) { + for (DataType dType : DataType.stringTypes()) { suppliers.add(new TestCaseSupplier("split test " + sType.toString() + " " + dType.toString(), List.of(sType, dType), () -> { String delimiter = randomAlphaOfLength(1); List strings = IntStream.range(0, between(1, 5)) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java index 41d1d3840eb98..b004adca351ab 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java @@ -203,8 +203,8 @@ private static void bytesRefs(List suppliers, int items) { ); })); - for (DataType type1 : new DataType[] { DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT }) { - for (DataType type2 : new DataType[] { DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT }) { + for (DataType type1 : DataType.stringTypes()) { + for (DataType type2 : DataType.stringTypes()) { if (type1 == type2 || items > 1) continue; suppliers.add(new TestCaseSupplier(type1 + " " + type2, List.of(type1, type2), () -> { List inlist = randomList(items, items, () -> randomLiteral(type1).value()); From c2b80f3148c75811aee3b5da4773b35575360378 Mon Sep 17 00:00:00 2001 From: Ioana Tagirta Date: Tue, 22 Oct 2024 12:42:44 +0200 Subject: [PATCH 07/10] Add tests for conditional functions and expressions --- .../main/resources/mapping-semantic_text.json | 4 + .../src/main/resources/semantic_text.csv | 8 +- .../src/main/resources/semantic_text.csv-spec | 92 ++++++++++++++++++- .../scalar/conditional/CaseTests.java | 1 + .../scalar/conditional/GreatestTests.java | 30 +++--- .../scalar/conditional/LeastTests.java | 30 +++--- .../scalar/convert/FromBase64Tests.java | 30 +++--- .../scalar/convert/ToBase64Tests.java | 30 +++--- 8 files changed, 152 insertions(+), 73 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json index b110d6fd4cdd5..c587b69828170 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-semantic_text.json @@ -68,6 +68,10 @@ }, "value": { "type": "long" + }, + "st_base64": { + "type": "semantic_text", + "inference_id": "test_sparse_inference" } } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv index c6de9a208e9a7..6cae82cfefa0a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv @@ -1,4 +1,4 @@ -_id:keyword,semantic_text_field:semantic_text,st_bool:semantic_text,st_cartesian_point:semantic_text,st_cartesian_shape:semantic_text,st_datetime:semantic_text,st_double:semantic_text,st_geopoint:semantic_text,st_geoshape:semantic_text,st_integer:semantic_text,st_ip:semantic_text,st_long:semantic_text,st_unsigned_long:semantic_text,st_version:semantic_text,st_multi_value:semantic_text,st_unicode:semantic_text,host:keyword,description:text,value:long -1,live long and prosper,false,"POINT(4297.11 -1475.53)",,1953-09-02T00:00:00.000Z,5.20128E11,"POINT(42.97109630194 14.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",23,1.1.1.1,2147483648,2147483648,1.2.3,["Hello there!", "This is a random value", "for testing purposes"],你吃饭了吗,"host1","some description1",1001 -2,all we have to decide is what to do with the time that is given to us,true,"POINT(7580.93 2272.77)",,2023-09-24T15:57:00.000Z,4541.11,"POINT(37.97109630194 21.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",122,1.1.2.1,123,2147483648.2,9.0.0,["nice to meet you", "bye bye!"],["谢谢", "对不起我的中文不好"],"host2","some description2",1002 -3,be excellent to each other,,,,,,,,,,,,,,,"host3","some description3",1003 +_id:keyword,semantic_text_field:semantic_text,st_bool:semantic_text,st_cartesian_point:semantic_text,st_cartesian_shape:semantic_text,st_datetime:semantic_text,st_double:semantic_text,st_geopoint:semantic_text,st_geoshape:semantic_text,st_integer:semantic_text,st_ip:semantic_text,st_long:semantic_text,st_unsigned_long:semantic_text,st_version:semantic_text,st_multi_value:semantic_text,st_unicode:semantic_text,host:keyword,description:text,value:long,st_base64:semantic_text +1,live long and prosper,false,"POINT(4297.11 -1475.53)",,1953-09-02T00:00:00.000Z,5.20128E11,"POINT(42.97109630194 14.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",23,1.1.1.1,2147483648,2147483648,1.2.3,["Hello there!", "This is a random value", "for testing purposes"],你吃饭了吗,"host1","some description1",1001,ZWxhc3RpYw== +2,all we have to decide is what to do with the time that is given to us,true,"POINT(7580.93 2272.77)",,2023-09-24T15:57:00.000Z,4541.11,"POINT(37.97109630194 21.7552534413725)","POLYGON ((30 10\, 40 40\, 20 40\, 10 20\, 30 10))",122,1.1.2.1,123,2147483648.2,9.0.0,["nice to meet you", "bye bye!"],["谢谢", "对不起我的中文不好"],"host2","some description2",1002,aGVsbG8= +3,be excellent to each other,,,,,,,,,,,,,,,"host3","some description3",1003, diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec index dfeb3f1f44dac..8626998778243 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec @@ -174,6 +174,66 @@ host:keyword | semantic_text_field:semantic_text "host3" | be excellent to each other ; +case +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = case(st_ip == "1.1.1.1", "okay", "try again") +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:keyword +1 | okay +2 | try again +3 | try again +; + +coalesce +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = coalesce(st_version, st_ip, semantic_text_field) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:semantic_text +1 | 1.2.3 +2 | 9.0.0 +3 | be excellent to each other +; + +greatest +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = greatest(semantic_text_field, st_version) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:semantic_text +1 | live long and prosper +2 | all we have to decide is what to do with the time that is given to us +3 | null +; + +least +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = least(semantic_text_field, st_version) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:semantic_text +1 | 1.2.3 +2 | 9.0.0 +3 | null +; + convertToBool required_capability: semantic_text_type @@ -383,6 +443,20 @@ be excellent to each other live long and prosper ; +fromBase64 +required_capability: semantic_text_type +FROM semantic_text +| EVAL result = from_base64(st_base64) +| SORT result +| KEEP result +; + +result:keyword +elastic +hello +null +; + left required_capability: semantic_text_type @@ -546,6 +620,21 @@ _id:keyword | result:keyword 3 | e ; +toBase64 +required_capability: semantic_text_type + +FROM semantic_text METADATA _id +| EVAL result = to_base64(st_integer) +| KEEP _id, result +| SORT _id +; + +_id:keyword | result:keyword +1 | MjM= +2 | MTIy +3 | null +; + toLower required_capability: semantic_text_type @@ -863,7 +952,6 @@ _id: keyword | result:bool 3 | true ; - lessThanOrEqualToBetweenFields required_capability: semantic_text_type @@ -879,7 +967,6 @@ _id: keyword | result:bool 3 | null ; - greaterThanWithConstant required_capability: semantic_text_type @@ -895,7 +982,6 @@ _id: keyword | result:bool 3 | false ; - greaterThanBetweenFields required_capability: semantic_text_type diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java index db3fce244c9a8..23fd7456a73ea 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java @@ -60,6 +60,7 @@ public class CaseTests extends AbstractScalarFunctionTestCase { ).collect(Collectors.toList()); if (Build.current().isSnapshot()) { t.add(DataType.DATE_NANOS); + t.add(DataType.SEMANTIC_TEXT); } TYPES = unmodifiableList(t); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java index 311e3e3d89149..f2612ae8c77f3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestTests.java @@ -40,21 +40,23 @@ public static Iterable parameters() { builder.expectFlattenedInt(IntStream::max); builder.expectFlattenedLong(LongStream::max); List suppliers = builder.suppliers(); - suppliers.add( - new TestCaseSupplier( - "(a, b)", - List.of(DataType.KEYWORD, DataType.KEYWORD), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("a"), DataType.KEYWORD, "a"), - new TestCaseSupplier.TypedData(new BytesRef("b"), DataType.KEYWORD, "b") - ), - "GreatestBytesRefEvaluator[values=[MvMax[field=Attribute[channel=0]], MvMax[field=Attribute[channel=1]]]]", - DataType.KEYWORD, - equalTo(new BytesRef("b")) + for (DataType stringType : DataType.stringTypes()) { + suppliers.add( + new TestCaseSupplier( + "(a, b)", + List.of(stringType, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("a"), stringType, "a"), + new TestCaseSupplier.TypedData(new BytesRef("b"), stringType, "b") + ), + "GreatestBytesRefEvaluator[values=[MvMax[field=Attribute[channel=0]], MvMax[field=Attribute[channel=1]]]]", + stringType, + equalTo(new BytesRef("b")) + ) ) - ) - ); + ); + } suppliers.add( new TestCaseSupplier( "(a, b)", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java index 69842fde90312..31f7e5f785c9c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastTests.java @@ -39,21 +39,23 @@ public static Iterable parameters() { builder.expectFlattenedInt(IntStream::min); builder.expectFlattenedLong(LongStream::min); List suppliers = builder.suppliers(); - suppliers.add( - new TestCaseSupplier( - "(a, b)", - List.of(DataType.KEYWORD, DataType.KEYWORD), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("a"), DataType.KEYWORD, "a"), - new TestCaseSupplier.TypedData(new BytesRef("b"), DataType.KEYWORD, "b") - ), - "LeastBytesRefEvaluator[values=[MvMin[field=Attribute[channel=0]], MvMin[field=Attribute[channel=1]]]]", - DataType.KEYWORD, - equalTo(new BytesRef("a")) + for (DataType stringType : DataType.stringTypes()) { + suppliers.add( + new TestCaseSupplier( + "(a, b)", + List.of(stringType, stringType), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(new BytesRef("a"), stringType, "a"), + new TestCaseSupplier.TypedData(new BytesRef("b"), stringType, "b") + ), + "LeastBytesRefEvaluator[values=[MvMin[field=Attribute[channel=0]], MvMin[field=Attribute[channel=1]]]]", + stringType, + equalTo(new BytesRef("a")) + ) ) - ) - ); + ); + } suppliers.add( new TestCaseSupplier( "(a, b)", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java index f472e5ef5efd9..60901e2a8214f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java @@ -35,25 +35,17 @@ public FromBase64Tests(@Name("TestCase") Supplier tes @ParametersFactory public static Iterable parameters() { List suppliers = new ArrayList<>(); - suppliers.add(new TestCaseSupplier(List.of(DataType.KEYWORD), () -> { - BytesRef input = new BytesRef(randomAlphaOfLength(6)); - return new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(input, DataType.KEYWORD, "string")), - "FromBase64Evaluator[field=Attribute[channel=0]]", - DataType.KEYWORD, - equalTo(new BytesRef(Base64.getDecoder().decode(input.utf8ToString().getBytes(StandardCharsets.UTF_8)))) - ); - })); - - suppliers.add(new TestCaseSupplier(List.of(DataType.TEXT), () -> { - BytesRef input = new BytesRef(randomAlphaOfLength(54)); - return new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(input, DataType.TEXT, "string")), - "FromBase64Evaluator[field=Attribute[channel=0]]", - DataType.KEYWORD, - equalTo(new BytesRef(Base64.getDecoder().decode(input.utf8ToString().getBytes(StandardCharsets.UTF_8)))) - ); - })); + for (DataType dataType : DataType.stringTypes()) { + suppliers.add(new TestCaseSupplier(List.of(dataType), () -> { + BytesRef input = new BytesRef(randomAlphaOfLength(54)); + return new TestCaseSupplier.TestCase( + List.of(new TestCaseSupplier.TypedData(input, dataType, "string")), + "FromBase64Evaluator[field=Attribute[channel=0]]", + DataType.KEYWORD, + equalTo(new BytesRef(Base64.getDecoder().decode(input.utf8ToString().getBytes(StandardCharsets.UTF_8)))) + ); + })); + } return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java index 06b5af8d7067b..6e6ff7bf52fce 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java @@ -36,25 +36,17 @@ public ToBase64Tests(@Name("TestCase") Supplier testC @ParametersFactory public static Iterable parameters() { List suppliers = new ArrayList<>(); - suppliers.add(new TestCaseSupplier(List.of(DataType.KEYWORD), () -> { - BytesRef input = (BytesRef) randomLiteral(DataType.KEYWORD).value(); - return new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(input, DataType.KEYWORD, "string")), - "ToBase64Evaluator[field=Attribute[channel=0]]", - DataType.KEYWORD, - equalTo(new BytesRef(Base64.getEncoder().encode(input.utf8ToString().getBytes(StandardCharsets.UTF_8)))) - ); - })); - - suppliers.add(new TestCaseSupplier(List.of(DataType.TEXT), () -> { - BytesRef input = (BytesRef) randomLiteral(DataType.TEXT).value(); - return new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(input, DataType.TEXT, "string")), - "ToBase64Evaluator[field=Attribute[channel=0]]", - DataType.KEYWORD, - equalTo(new BytesRef(Base64.getEncoder().encode(input.utf8ToString().getBytes(StandardCharsets.UTF_8)))) - ); - })); + for (DataType dataType : DataType.stringTypes()) { + suppliers.add(new TestCaseSupplier(List.of(dataType), () -> { + BytesRef input = (BytesRef) randomLiteral(dataType).value(); + return new TestCaseSupplier.TestCase( + List.of(new TestCaseSupplier.TypedData(input, dataType, "string")), + "ToBase64Evaluator[field=Attribute[channel=0]]", + DataType.KEYWORD, + equalTo(new BytesRef(Base64.getEncoder().encode(input.utf8ToString().getBytes(StandardCharsets.UTF_8)))) + ); + })); + } return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); } From 15501f08a507e2902faa7de7c680c37822066e52 Mon Sep 17 00:00:00 2001 From: Ioana Tagirta Date: Tue, 29 Oct 2024 14:20:13 +0100 Subject: [PATCH 08/10] Fix tests after merge --- .../xpack/esql/core/type/DataType.java | 2 +- .../src/main/resources/semantic_text.csv-spec | 32 +++++++++---------- .../function/scalar/convert/ToDateNanos.java | 2 ++ .../esql/type/EsqlDataTypeConverter.java | 6 ++-- .../function/AbstractFunctionTestCase.java | 4 +-- .../scalar/conditional/CaseTests.java | 3 +- .../function/scalar/string/ConcatTests.java | 2 +- .../esql/type/EsqlDataTypeConverterTests.java | 4 +++ 8 files changed, 30 insertions(+), 25 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 8c745af0e75bd..81739536c6572 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -588,7 +588,7 @@ static Builder builder() { } public DataType noText() { - return this == TEXT ? KEYWORD : this; + return isString(this) ? KEYWORD : this; } /** diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec index 8626998778243..de2a79df06a50 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/semantic_text.csv-spec @@ -198,7 +198,7 @@ FROM semantic_text METADATA _id | SORT _id ; -_id:keyword | result:semantic_text +_id:keyword | result:keyword 1 | 1.2.3 2 | 9.0.0 3 | be excellent to each other @@ -213,7 +213,7 @@ FROM semantic_text METADATA _id | SORT _id ; -_id:keyword | result:semantic_text +_id:keyword | result:keyword 1 | live long and prosper 2 | all we have to decide is what to do with the time that is given to us 3 | null @@ -228,7 +228,7 @@ FROM semantic_text METADATA _id | SORT _id ; -_id:keyword | result:semantic_text +_id:keyword | result:keyword 1 | 1.2.3 2 | 9.0.0 3 | null @@ -511,7 +511,7 @@ FROM semantic_text | KEEP result ; -result:semantic_text +result:keyword all we have to decide is what to do with the time that is given to us be excellent to each other live long and prosper @@ -569,7 +569,7 @@ FROM semantic_text | SORT result ; -result:semantic_text +result:keyword all we have to decide is what to do with the time that is given to us be excellent to each other live long and prosper @@ -644,7 +644,7 @@ FROM semantic_text METADATA _id | SORT _id ; -_id:keyword | result:semantic_text +_id:keyword | result:keyword 1 | point(4297.11 -1475.53) 2 | point(7580.93 2272.77) 3 | null @@ -659,7 +659,7 @@ FROM semantic_text METADATA _id | SORT _id ; -_id:keyword | result:semantic_text +_id:keyword | result:keyword 1 | LIVE LONG AND PROSPER 2 | ALL WE HAVE TO DECIDE IS WHAT TO DO WITH THE TIME THAT IS GIVEN TO US 3 | BE EXCELLENT TO EACH OTHER @@ -674,7 +674,7 @@ FROM semantic_text | KEEP result ; -result:semantic_text +result:keyword all we have to decide is what to do with the time that is given to us be excellent to each other live long and prosper @@ -689,7 +689,7 @@ FROM semantic_text METADATA _id | SORT _id ; -_id: keyword | result:semantic_text +_id: keyword | result:keyword 1 | ["Hello there!", "This is a random value", "for testing purposes", "2147483648"] 2 | ["nice to meet you", "bye bye!", "123"] 3 | null @@ -734,7 +734,7 @@ FROM semantic_text METADATA _id | SORT _id ; -_id: keyword | result:semantic_text +_id: keyword | result:keyword 1 | ["Hello there!", "This is a random value", "for testing purposes"] 2 | ["nice to meet you", "bye bye!"] 3 | null @@ -749,7 +749,7 @@ FROM semantic_text METADATA _id | SORT _id ; -_id: keyword | result:semantic_text +_id: keyword | result:keyword 1 | Hello there! 2 | nice to meet you 3 | null @@ -764,7 +764,7 @@ FROM semantic_text METADATA _id | SORT _id ; -_id: keyword | result:semantic_text +_id: keyword | result:keyword 1 | for testing purposes 2 | bye bye! 3 | null @@ -779,7 +779,7 @@ FROM semantic_text METADATA _id | SORT _id ; -_id: keyword | result:semantic_text +_id: keyword | result:keyword 1 | for testing purposes 2 | nice to meet you 3 | null @@ -794,7 +794,7 @@ FROM semantic_text METADATA _id | SORT _id ; -_id: keyword | result:semantic_text +_id: keyword | result:keyword 1 | Hello there! 2 | bye bye! 3 | null @@ -809,7 +809,7 @@ FROM semantic_text METADATA _id | SORT _id ; -_id: keyword | result:semantic_text +_id: keyword | result:keyword 1 | ["This is a random value", "for testing purposes"] 2 | bye bye! 3 | null @@ -824,7 +824,7 @@ FROM semantic_text METADATA _id | SORT _id ; -_id: keyword | result:semantic_text +_id: keyword | result:keyword 1 | ["Hello there!", "This is a random value", "for testing purposes"] 2 | ["bye bye!", "nice to meet you"] 3 | null diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java index 9a6a91b7ccedd..8c4375b424cdc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java @@ -32,6 +32,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_NANOS_FORMATTER; @@ -49,6 +50,7 @@ public class ToDateNanos extends AbstractConvertFunction { Map.entry(LONG, ToDateNanosFromLongEvaluator.Factory::new), Map.entry(KEYWORD, ToDateNanosFromStringEvaluator.Factory::new), Map.entry(TEXT, ToDateNanosFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToDateNanosFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToDateNanosFromDoubleEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToLongFromUnsignedLongEvaluator.Factory::new) /* diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index 6943a84e43491..7fb998e82001e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -366,10 +366,10 @@ public static DataType commonType(DataType left, DataType right) { } } if (isString(left) && isString(right)) { - if (left == TEXT || right == TEXT) { - return TEXT; - } if (left == SEMANTIC_TEXT || right == SEMANTIC_TEXT) { + return KEYWORD; + } + if (left == TEXT || right == TEXT) { return TEXT; } return right; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index 90a4e11d2e3e4..c05f8e0990b3c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -241,8 +241,8 @@ protected static List anyNullIsNull( .mapToObj( i -> i == finalNullPosition ? (oc.getData().get(i).isMultiRow() - ? TestCaseSupplier.TypedData.MULTI_ROW_NULL - : TestCaseSupplier.TypedData.NULL) + ? TestCaseSupplier.TypedData.MULTI_ROW_NULL + : TestCaseSupplier.TypedData.NULL) : oc.getData().get(i) ) .toList(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java index fca50b46c4822..93322ab0e35a7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java @@ -60,8 +60,7 @@ public class CaseTests extends AbstractScalarFunctionTestCase { DataType.NULL ).collect(Collectors.toList()); if (Build.current().isSnapshot()) { - t.add(DataType.DATE_NANOS); - t.add(DataType.SEMANTIC_TEXT); + t.addAll(DataType.UNDER_CONSTRUCTION.keySet()); } TYPES = unmodifiableList(t); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java index 3cf855568ef49..42c6284a3c25a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatTests.java @@ -100,7 +100,7 @@ private static TestCaseSupplier supplier(String name, DataType type, int length, private static void add(List suppliers, String name, int length, Supplier valueSupplier) { Map>> permutations = new HashMap>>(); - List supportedDataTypes = List.of(DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT); + List supportedDataTypes = DataType.stringTypes().stream().toList(); permutations.put(0, List.of(List.of(DataType.KEYWORD), List.of(DataType.TEXT))); for (int v = 0; v < length - 1; v++) { List> current = permutations.get(v); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java index babb9fc8c0bd1..b2228b5543ef2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java @@ -30,11 +30,13 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.HALF_FLOAT; import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.IP; +import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT; import static org.elasticsearch.xpack.esql.core.type.DataType.PARTIAL_AGG; import static org.elasticsearch.xpack.esql.core.type.DataType.SCALED_FLOAT; +import static org.elasticsearch.xpack.esql.core.type.DataType.SEMANTIC_TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.SHORT; import static org.elasticsearch.xpack.esql.core.type.DataType.SOURCE; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; @@ -69,6 +71,8 @@ public void testCommonTypeStrings() { } else if ((isString(dataType1) && isString(dataType2))) { if (dataType1 == dataType2) { assertEqualsCommonType(dataType1, dataType2, dataType1); + } else if (dataType1 == SEMANTIC_TEXT || dataType2 == SEMANTIC_TEXT) { + assertEqualsCommonType(dataType1, dataType2, KEYWORD); } else { assertEqualsCommonType(dataType1, dataType2, TEXT); } From 8bd0e3e45e7ecaf33a53b34bf0344f149c84ea61 Mon Sep 17 00:00:00 2001 From: Ioana Tagirta Date: Thu, 31 Oct 2024 15:52:43 +0100 Subject: [PATCH 09/10] Reorder semantic_text evaluators and tests --- .../function/scalar/convert/ToBoolean.java | 4 +-- .../scalar/convert/ToCartesianPoint.java | 4 +-- .../scalar/convert/ToCartesianShape.java | 4 +-- .../function/scalar/convert/ToDatetime.java | 4 +-- .../function/scalar/convert/ToDouble.java | 4 +-- .../function/scalar/convert/ToGeoPoint.java | 4 +-- .../function/scalar/convert/ToGeoShape.java | 4 +-- .../function/scalar/convert/ToInteger.java | 4 +-- .../function/scalar/convert/ToLong.java | 4 +-- .../function/scalar/convert/ToString.java | 4 +-- .../scalar/convert/ToUnsignedLong.java | 4 +-- .../predicate/operator/comparison/Equals.java | 4 +-- .../operator/comparison/GreaterThan.java | 4 +-- .../comparison/GreaterThanOrEqual.java | 4 +-- .../operator/comparison/LessThan.java | 4 +-- .../operator/comparison/LessThanOrEqual.java | 4 +-- .../operator/comparison/NotEquals.java | 4 +-- .../scalar/multivalue/MvAppendTests.java | 32 ++++++++--------- .../scalar/multivalue/MvSliceTests.java | 34 +++++++++---------- .../scalar/multivalue/MvSortTests.java | 28 +++++++-------- 20 files changed, 81 insertions(+), 81 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java index 58f706f01c49a..ad73de7829692 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBoolean.java @@ -45,11 +45,11 @@ public class ToBoolean extends AbstractConvertFunction { Map.entry(BOOLEAN, (field, source) -> field), Map.entry(KEYWORD, ToBooleanFromStringEvaluator.Factory::new), Map.entry(TEXT, ToBooleanFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToBooleanFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToBooleanFromDoubleEvaluator.Factory::new), Map.entry(LONG, ToBooleanFromLongEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToBooleanFromUnsignedLongEvaluator.Factory::new), - Map.entry(INTEGER, ToBooleanFromIntEvaluator.Factory::new), - Map.entry(SEMANTIC_TEXT, ToBooleanFromStringEvaluator.Factory::new) + Map.entry(INTEGER, ToBooleanFromIntEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java index db3617fe589eb..92ae2cd0ade52 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPoint.java @@ -39,8 +39,8 @@ public class ToCartesianPoint extends AbstractConvertFunction { private static final Map EVALUATORS = Map.ofEntries( Map.entry(CARTESIAN_POINT, (fieldEval, source) -> fieldEval), Map.entry(KEYWORD, ToCartesianPointFromStringEvaluator.Factory::new), - Map.entry(SEMANTIC_TEXT, ToCartesianPointFromStringEvaluator.Factory::new), - Map.entry(TEXT, ToCartesianPointFromStringEvaluator.Factory::new) + Map.entry(TEXT, ToCartesianPointFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToCartesianPointFromStringEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java index 5ec5d2a4ac741..83e66e9e3190f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShape.java @@ -41,8 +41,8 @@ public class ToCartesianShape extends AbstractConvertFunction { Map.entry(CARTESIAN_POINT, (fieldEval, source) -> fieldEval), Map.entry(CARTESIAN_SHAPE, (fieldEval, source) -> fieldEval), Map.entry(KEYWORD, ToCartesianShapeFromStringEvaluator.Factory::new), - Map.entry(SEMANTIC_TEXT, ToCartesianShapeFromStringEvaluator.Factory::new), - Map.entry(TEXT, ToCartesianShapeFromStringEvaluator.Factory::new) + Map.entry(TEXT, ToCartesianShapeFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToCartesianShapeFromStringEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java index 17dcde81ae330..f8fe663b9086c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java @@ -48,10 +48,10 @@ public class ToDatetime extends AbstractConvertFunction { Map.entry(LONG, (field, source) -> field), Map.entry(KEYWORD, ToDatetimeFromStringEvaluator.Factory::new), Map.entry(TEXT, ToDatetimeFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToDatetimeFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToLongFromDoubleEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToLongFromUnsignedLongEvaluator.Factory::new), - Map.entry(INTEGER, ToLongFromIntEvaluator.Factory::new), // CastIntToLongEvaluator would be a candidate, but not MV'd - Map.entry(SEMANTIC_TEXT, ToDatetimeFromStringEvaluator.Factory::new) + Map.entry(INTEGER, ToLongFromIntEvaluator.Factory::new) // CastIntToLongEvaluator would be a candidate, but not MV'd ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java index c0b671d32bb38..67b7af73576eb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java @@ -45,13 +45,13 @@ public class ToDouble extends AbstractConvertFunction { Map.entry(DATETIME, ToDoubleFromLongEvaluator.Factory::new), // CastLongToDoubleEvaluator would be a candidate, but not MV'd Map.entry(KEYWORD, ToDoubleFromStringEvaluator.Factory::new), Map.entry(TEXT, ToDoubleFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToDoubleFromStringEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToDoubleFromUnsignedLongEvaluator.Factory::new), Map.entry(LONG, ToDoubleFromLongEvaluator.Factory::new), // CastLongToDoubleEvaluator would be a candidate, but not MV'd Map.entry(INTEGER, ToDoubleFromIntEvaluator.Factory::new), // CastIntToDoubleEvaluator would be a candidate, but not MV'd Map.entry(DataType.COUNTER_DOUBLE, (field, source) -> field), Map.entry(DataType.COUNTER_INTEGER, ToDoubleFromIntEvaluator.Factory::new), - Map.entry(DataType.COUNTER_LONG, ToDoubleFromLongEvaluator.Factory::new), - Map.entry(SEMANTIC_TEXT, ToDoubleFromStringEvaluator.Factory::new) + Map.entry(DataType.COUNTER_LONG, ToDoubleFromLongEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java index 2dc5f1662b786..42af06a40553d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPoint.java @@ -39,8 +39,8 @@ public class ToGeoPoint extends AbstractConvertFunction { private static final Map EVALUATORS = Map.ofEntries( Map.entry(GEO_POINT, (fieldEval, source) -> fieldEval), Map.entry(KEYWORD, ToGeoPointFromStringEvaluator.Factory::new), - Map.entry(SEMANTIC_TEXT, ToGeoPointFromStringEvaluator.Factory::new), - Map.entry(TEXT, ToGeoPointFromStringEvaluator.Factory::new) + Map.entry(TEXT, ToGeoPointFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToGeoPointFromStringEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java index 3d6e3f3baa348..b5b6db2752b06 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShape.java @@ -41,8 +41,8 @@ public class ToGeoShape extends AbstractConvertFunction { Map.entry(GEO_POINT, (fieldEval, source) -> fieldEval), Map.entry(GEO_SHAPE, (fieldEval, source) -> fieldEval), Map.entry(KEYWORD, ToGeoShapeFromStringEvaluator.Factory::new), - Map.entry(SEMANTIC_TEXT, ToGeoShapeFromStringEvaluator.Factory::new), - Map.entry(TEXT, ToGeoShapeFromStringEvaluator.Factory::new) + Map.entry(TEXT, ToGeoShapeFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToGeoShapeFromStringEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java index 254cf5fe536de..d316b6eb46c38 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java @@ -51,11 +51,11 @@ public class ToInteger extends AbstractConvertFunction { Map.entry(DATETIME, ToIntegerFromLongEvaluator.Factory::new), Map.entry(KEYWORD, ToIntegerFromStringEvaluator.Factory::new), Map.entry(TEXT, ToIntegerFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToIntegerFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToIntegerFromDoubleEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToIntegerFromUnsignedLongEvaluator.Factory::new), Map.entry(LONG, ToIntegerFromLongEvaluator.Factory::new), - Map.entry(COUNTER_INTEGER, (fieldEval, source) -> fieldEval), - Map.entry(SEMANTIC_TEXT, ToIntegerFromStringEvaluator.Factory::new) + Map.entry(COUNTER_INTEGER, (fieldEval, source) -> fieldEval) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java index 1a4026e3a5cce..dbfb52b408b44 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java @@ -48,12 +48,12 @@ public class ToLong extends AbstractConvertFunction { Map.entry(BOOLEAN, ToLongFromBooleanEvaluator.Factory::new), Map.entry(KEYWORD, ToLongFromStringEvaluator.Factory::new), Map.entry(TEXT, ToLongFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToLongFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToLongFromDoubleEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToLongFromUnsignedLongEvaluator.Factory::new), Map.entry(INTEGER, ToLongFromIntEvaluator.Factory::new), // CastIntToLongEvaluator would be a candidate, but not MV'd Map.entry(DataType.COUNTER_LONG, (field, source) -> field), - Map.entry(DataType.COUNTER_INTEGER, ToLongFromIntEvaluator.Factory::new), - Map.entry(SEMANTIC_TEXT, ToLongFromStringEvaluator.Factory::new) + Map.entry(DataType.COUNTER_INTEGER, ToLongFromIntEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java index 5bf572d3bee3f..2c8ecd794ef0b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java @@ -61,13 +61,13 @@ public class ToString extends AbstractConvertFunction implements EvaluatorMapper Map.entry(LONG, ToStringFromLongEvaluator.Factory::new), Map.entry(INTEGER, ToStringFromIntEvaluator.Factory::new), Map.entry(TEXT, (fieldEval, source) -> fieldEval), + Map.entry(SEMANTIC_TEXT, (fieldEval, source) -> fieldEval), Map.entry(VERSION, ToStringFromVersionEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToStringFromUnsignedLongEvaluator.Factory::new), Map.entry(GEO_POINT, ToStringFromGeoPointEvaluator.Factory::new), Map.entry(CARTESIAN_POINT, ToStringFromCartesianPointEvaluator.Factory::new), Map.entry(CARTESIAN_SHAPE, ToStringFromCartesianShapeEvaluator.Factory::new), - Map.entry(GEO_SHAPE, ToStringFromGeoShapeEvaluator.Factory::new), - Map.entry(SEMANTIC_TEXT, (fieldEval, source) -> fieldEval) + Map.entry(GEO_SHAPE, ToStringFromGeoShapeEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java index 82df3a0589255..ea06793f7adb6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLong.java @@ -52,10 +52,10 @@ public class ToUnsignedLong extends AbstractConvertFunction { Map.entry(BOOLEAN, ToUnsignedLongFromBooleanEvaluator.Factory::new), Map.entry(KEYWORD, ToUnsignedLongFromStringEvaluator.Factory::new), Map.entry(TEXT, ToUnsignedLongFromStringEvaluator.Factory::new), + Map.entry(SEMANTIC_TEXT, ToUnsignedLongFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToUnsignedLongFromDoubleEvaluator.Factory::new), Map.entry(LONG, ToUnsignedLongFromLongEvaluator.Factory::new), - Map.entry(INTEGER, ToUnsignedLongFromIntEvaluator.Factory::new), - Map.entry(SEMANTIC_TEXT, ToUnsignedLongFromStringEvaluator.Factory::new) + Map.entry(INTEGER, ToUnsignedLongFromIntEvaluator.Factory::new) ); @FunctionInfo( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java index 730653e53dc14..6bb249385affe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java @@ -42,9 +42,9 @@ public class Equals extends EsqlBinaryComparison implements Negatable suppliers) { ); })); + suppliers.add(new TestCaseSupplier(List.of(DataType.SEMANTIC_TEXT, DataType.SEMANTIC_TEXT), () -> { + List field1 = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); + List field2 = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); + var result = new ArrayList<>(field1); + result.addAll(field2); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field1, DataType.SEMANTIC_TEXT, "field1"), + new TestCaseSupplier.TypedData(field2, DataType.SEMANTIC_TEXT, "field2") + ), + "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", + DataType.SEMANTIC_TEXT, + equalTo(result) + ); + })); + suppliers.add(new TestCaseSupplier(List.of(DataType.IP, DataType.IP), () -> { List field1 = randomList(1, 10, () -> randomLiteral(DataType.IP).value()); List field2 = randomList(1, 10, () -> randomLiteral(DataType.IP).value()); @@ -266,22 +282,6 @@ private static void bytesRefs(List suppliers) { equalTo(result) ); })); - - suppliers.add(new TestCaseSupplier(List.of(DataType.SEMANTIC_TEXT, DataType.SEMANTIC_TEXT), () -> { - List field1 = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); - List field2 = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); - var result = new ArrayList<>(field1); - result.addAll(field2); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(field1, DataType.SEMANTIC_TEXT, "field1"), - new TestCaseSupplier.TypedData(field2, DataType.SEMANTIC_TEXT, "field2") - ), - "MvAppendBytesRefEvaluator[field1=Attribute[channel=0], field2=Attribute[channel=1]]", - DataType.SEMANTIC_TEXT, - equalTo(result) - ); - })); } private static void nulls(List suppliers) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java index 1cb7b7ea46aa4..d5284602bf40c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java @@ -255,6 +255,23 @@ private static void bytesRefs(List suppliers) { ); })); + suppliers.add(new TestCaseSupplier(List.of(DataType.SEMANTIC_TEXT, DataType.INTEGER, DataType.INTEGER), () -> { + List field = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); + int length = field.size(); + int start = randomIntBetween(0, length - 1); + int end = randomIntBetween(start, length - 1); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field, DataType.SEMANTIC_TEXT, "field"), + new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"), + new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end") + ), + "MvSliceBytesRefEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]", + DataType.SEMANTIC_TEXT, + equalTo(start == end ? field.get(start) : field.subList(start, end + 1)) + ); + })); + suppliers.add(new TestCaseSupplier(List.of(DataType.IP, DataType.INTEGER, DataType.INTEGER), () -> { List field = randomList(1, 10, () -> randomLiteral(DataType.IP).value()); int length = field.size(); @@ -356,22 +373,5 @@ private static void bytesRefs(List suppliers) { equalTo(start == end ? field.get(start) : field.subList(start, end + 1)) ); })); - - suppliers.add(new TestCaseSupplier(List.of(DataType.SEMANTIC_TEXT, DataType.INTEGER, DataType.INTEGER), () -> { - List field = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); - int length = field.size(); - int start = randomIntBetween(0, length - 1); - int end = randomIntBetween(start, length - 1); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(field, DataType.SEMANTIC_TEXT, "field"), - new TestCaseSupplier.TypedData(start, DataType.INTEGER, "start"), - new TestCaseSupplier.TypedData(end, DataType.INTEGER, "end") - ), - "MvSliceBytesRefEvaluator[field=Attribute[channel=0], start=Attribute[channel=1], end=Attribute[channel=2]]", - DataType.SEMANTIC_TEXT, - equalTo(start == end ? field.get(start) : field.subList(start, end + 1)) - ); - })); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java index f4271b01e75cd..e5f240c811bd0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java @@ -171,6 +171,20 @@ private static void bytesRefs(List suppliers) { ); })); + suppliers.add(new TestCaseSupplier(List.of(DataType.SEMANTIC_TEXT, DataType.KEYWORD), () -> { + List field = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); + BytesRef order = new BytesRef("ASC"); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field, DataType.SEMANTIC_TEXT, "field"), + new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral() + ), + "MvSortBytesRef[field=Attribute[channel=0], order=true]", + DataType.SEMANTIC_TEXT, + equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList()) + ); + })); + suppliers.add(new TestCaseSupplier(List.of(DataType.IP, DataType.KEYWORD), () -> { List field = randomList(1, 10, () -> randomLiteral(DataType.IP).value()); BytesRef order = new BytesRef("DESC"); @@ -198,20 +212,6 @@ private static void bytesRefs(List suppliers) { equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList()) ); })); - - suppliers.add(new TestCaseSupplier(List.of(DataType.SEMANTIC_TEXT, DataType.KEYWORD), () -> { - List field = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); - BytesRef order = new BytesRef("ASC"); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(field, DataType.SEMANTIC_TEXT, "field"), - new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral() - ), - "MvSortBytesRef[field=Attribute[channel=0], order=true]", - DataType.SEMANTIC_TEXT, - equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList()) - ); - })); } private static void nulls(List suppliers) { From f41d0768c619686b7d2120fb75caa9d39b8bfac5 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 31 Oct 2024 16:45:02 +0100 Subject: [PATCH 10/10] Re-ordered two more places for SEMANTIC_TEXT after TEXT --- .../org/elasticsearch/xpack/esql/analysis/Verifier.java | 6 +++--- .../multivalue/AbstractMultivalueFunctionTestCase.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 82909ecdde46c..994ea3ecdbb0d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -506,6 +506,9 @@ public static Failure validateBinaryComparison(BinaryComparison bc) { List allowed = new ArrayList<>(); allowed.add(DataType.KEYWORD); allowed.add(DataType.TEXT); + if (EsqlCapabilities.Cap.SEMANTIC_TEXT_TYPE.isEnabled()) { + allowed.add(DataType.SEMANTIC_TEXT); + } allowed.add(DataType.IP); allowed.add(DataType.DATETIME); allowed.add(DataType.VERSION); @@ -513,9 +516,6 @@ public static Failure validateBinaryComparison(BinaryComparison bc) { allowed.add(DataType.GEO_SHAPE); allowed.add(DataType.CARTESIAN_POINT); allowed.add(DataType.CARTESIAN_SHAPE); - if (EsqlCapabilities.Cap.SEMANTIC_TEXT_TYPE.isEnabled()) { - allowed.add(DataType.SEMANTIC_TEXT); - } if (bc instanceof Equals || bc instanceof NotEquals) { allowed.add(DataType.BOOLEAN); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java index 2ff4f5dd87b30..65f5653f27e1a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java @@ -123,7 +123,7 @@ protected static void bytesRefs( Function expectedDataType, BiFunction, Matcher> matcher ) { - for (DataType type : new DataType[] { DataType.KEYWORD, DataType.TEXT, DataType.IP, DataType.VERSION, DataType.SEMANTIC_TEXT }) { + for (DataType type : new DataType[] { DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT, DataType.IP, DataType.VERSION }) { if (type != DataType.IP) { cases.add( new TestCaseSupplier(