From f1981e8db026a184d15dd28f501241b5b23baa9f Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 24 Apr 2024 11:04:49 -0700 Subject: [PATCH 01/13] Support counter type in ESQL --- .../kibana/definition/to_double.json | 12 +++++ .../kibana/definition/to_integer.json | 12 +++++ .../functions/kibana/definition/to_long.json | 12 +++++ .../esql/functions/types/to_double.asciidoc | 1 + .../esql/functions/types/to_integer.asciidoc | 1 + .../esql/functions/types/to_long.asciidoc | 1 + .../index/mapper/NumberFieldMapper.java | 4 -- .../src/main/resources/meta.csv-spec | 20 +++---- .../xpack/esql/action/PositionToXContent.java | 6 +-- .../xpack/esql/action/ResponseValueUtils.java | 6 +-- .../function/scalar/convert/ToDouble.java | 6 ++- .../function/scalar/convert/ToInteger.java | 6 ++- .../function/scalar/convert/ToLong.java | 6 ++- .../esql/planner/LocalExecutionPlanner.java | 3 +- .../xpack/esql/planner/PlannerUtils.java | 9 ++-- .../xpack/esql/plugin/EsqlFeatures.java | 8 ++- .../xpack/esql/type/EsqlDataTypeRegistry.java | 7 ++- .../xpack/esql/type/EsqlDataTypes.java | 20 ++++++- .../esql/action/EsqlQueryResponseTests.java | 5 +- .../function/AbstractFunctionTestCase.java | 52 +++++++++++++++++-- .../scalar/convert/ToDoubleTests.java | 11 ++++ .../scalar/convert/ToIntegerTests.java | 10 ++++ .../function/scalar/convert/ToLongTests.java | 10 ++++ .../esql/type/EsqlDataTypeRegistryTests.java | 6 ++- .../rest-api-spec/test/esql/40_tsdb.yml | 39 +++++++++----- 25 files changed, 218 insertions(+), 55 deletions(-) diff --git a/docs/reference/esql/functions/kibana/definition/to_double.json b/docs/reference/esql/functions/kibana/definition/to_double.json index 4a466e76562e9..362fd1863a68f 100644 --- a/docs/reference/esql/functions/kibana/definition/to_double.json +++ b/docs/reference/esql/functions/kibana/definition/to_double.json @@ -16,6 +16,18 @@ "variadic" : false, "returnType" : "double" }, + { + "params" : [ + { + "name" : "field", + "type" : "counter_double", + "optional" : false, + "description" : "Input value. The input can be a single- or multi-valued column or an expression." + } + ], + "variadic" : false, + "returnType" : "double" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/kibana/definition/to_integer.json b/docs/reference/esql/functions/kibana/definition/to_integer.json index 4284265c4f93c..2776d8b29c412 100644 --- a/docs/reference/esql/functions/kibana/definition/to_integer.json +++ b/docs/reference/esql/functions/kibana/definition/to_integer.json @@ -16,6 +16,18 @@ "variadic" : false, "returnType" : "integer" }, + { + "params" : [ + { + "name" : "field", + "type" : "counter_integer", + "optional" : false, + "description" : "Input value. The input can be a single- or multi-valued column or an expression." + } + ], + "variadic" : false, + "returnType" : "integer" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/kibana/definition/to_long.json b/docs/reference/esql/functions/kibana/definition/to_long.json index 25e7f82f18547..466de6800509d 100644 --- a/docs/reference/esql/functions/kibana/definition/to_long.json +++ b/docs/reference/esql/functions/kibana/definition/to_long.json @@ -16,6 +16,18 @@ "variadic" : false, "returnType" : "long" }, + { + "params" : [ + { + "name" : "field", + "type" : "counter_long", + "optional" : false, + "description" : "Input value. The input can be a single- or multi-valued column or an expression." + } + ], + "variadic" : false, + "returnType" : "long" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/types/to_double.asciidoc b/docs/reference/esql/functions/types/to_double.asciidoc index c78c3974af5a5..a9adcfa2eeb42 100644 --- a/docs/reference/esql/functions/types/to_double.asciidoc +++ b/docs/reference/esql/functions/types/to_double.asciidoc @@ -6,6 +6,7 @@ |=== field | result boolean | double +counter_double | double datetime | double double | double integer | double diff --git a/docs/reference/esql/functions/types/to_integer.asciidoc b/docs/reference/esql/functions/types/to_integer.asciidoc index 11fd7914c5b0f..974f3c9c82d88 100644 --- a/docs/reference/esql/functions/types/to_integer.asciidoc +++ b/docs/reference/esql/functions/types/to_integer.asciidoc @@ -6,6 +6,7 @@ |=== field | result boolean | integer +counter_integer | integer datetime | integer double | integer integer | integer diff --git a/docs/reference/esql/functions/types/to_long.asciidoc b/docs/reference/esql/functions/types/to_long.asciidoc index 4bc927fd94697..bb198dfa5abb9 100644 --- a/docs/reference/esql/functions/types/to_long.asciidoc +++ b/docs/reference/esql/functions/types/to_long.asciidoc @@ -6,6 +6,7 @@ |=== field | result boolean | long +counter_long | long datetime | long double | long integer | long diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index ebb6672cbab18..493d09a047a53 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1709,10 +1709,6 @@ public Function pointReaderIfPossible() { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - if (indexMode == IndexMode.TIME_SERIES && metricType == TimeSeriesParams.MetricType.COUNTER) { - // Counters are not supported by ESQL so we load them in null - return BlockLoader.CONSTANT_NULLS; - } if (hasDocValues()) { return type.blockLoaderFromDocValues(name()); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec index 93ecb003685bc..9f133c140853b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec @@ -85,16 +85,16 @@ double tau() "cartesian_point to_cartesianpoint(field:cartesian_point|keyword|text)" "cartesian_shape to_cartesianshape(field:cartesian_point|cartesian_shape|keyword|text)" "date to_datetime(field:date|keyword|text|double|long|unsigned_long|integer)" -"double to_dbl(field:boolean|date|keyword|text|double|long|unsigned_long|integer)" +"double to_dbl(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double)" "double to_degrees(number:double|integer|long|unsigned_long)" -"double to_double(field:boolean|date|keyword|text|double|long|unsigned_long|integer)" +"double to_double(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double)" "date to_dt(field:date|keyword|text|double|long|unsigned_long|integer)" "geo_point to_geopoint(field:geo_point|keyword|text)" "geo_shape to_geoshape(field:geo_point|geo_shape|keyword|text)" -"integer to_int(field:boolean|date|keyword|text|double|long|unsigned_long|integer)" -"integer to_integer(field:boolean|date|keyword|text|double|long|unsigned_long|integer)" +"integer to_int(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer)" +"integer to_integer(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer)" "ip to_ip(field:ip|keyword|text)" -"long to_long(field:boolean|date|keyword|text|double|long|unsigned_long|integer)" +"long to_long(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_long)" "keyword|text to_lower(str:keyword|text)" "double to_radians(number:double|integer|long|unsigned_long)" "keyword to_str(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)" @@ -198,16 +198,16 @@ to_boolean |field |"boolean|keyword|text|double to_cartesianpo|field |"cartesian_point|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. to_cartesiansh|field |"cartesian_point|cartesian_shape|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. to_datetime |field |"date|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. -to_dbl |field |"boolean|date|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. +to_dbl |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double" |Input value. The input can be a single- or multi-valued column or an expression. to_degrees |number |"double|integer|long|unsigned_long" |Input value. The input can be a single- or multi-valued column or an expression. -to_double |field |"boolean|date|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. +to_double |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double" |Input value. The input can be a single- or multi-valued column or an expression. to_dt |field |"date|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. to_geopoint |field |"geo_point|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. to_geoshape |field |"geo_point|geo_shape|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. -to_int |field |"boolean|date|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. -to_integer |field |"boolean|date|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. +to_int |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer" |Input value. The input can be a single- or multi-valued column or an expression. +to_integer |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer" |Input value. The input can be a single- or multi-valued column or an expression. to_ip |field |"ip|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. -to_long |field |"boolean|date|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. +to_long |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_long" |Input value. The input can be a single- or multi-valued column or an expression. to_lower |str |"keyword|text" |String expression. If `null`, the function returns `null`. to_radians |number |"double|integer|long|unsigned_long" |Input value. The input can be a single- or multi-valued column or an expression. to_str |field |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version" |Input value. The input can be a single- or multi-valued column or an expression. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java index 5488efda7834f..7e54bf94ac263 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java @@ -61,21 +61,21 @@ protected abstract XContentBuilder valueToXContent(XContentBuilder builder, ToXC public static PositionToXContent positionToXContent(ColumnInfo columnInfo, Block block, BytesRef scratch) { return switch (columnInfo.type()) { - case "long" -> new PositionToXContent(block) { + case "long", "counter_long" -> new PositionToXContent(block) { @Override protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Params params, int valueIndex) throws IOException { return builder.value(((LongBlock) block).getLong(valueIndex)); } }; - case "integer" -> new PositionToXContent(block) { + case "integer", "counter_integer" -> new PositionToXContent(block) { @Override protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Params params, int valueIndex) throws IOException { return builder.value(((IntBlock) block).getInt(valueIndex)); } }; - case "double" -> new PositionToXContent(block) { + case "double", "counter_double" -> new PositionToXContent(block) { @Override protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Params params, int valueIndex) throws IOException { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java index f467512fd6c0b..97b171fc0f070 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java @@ -123,9 +123,9 @@ static Object valueAtPosition(Block block, int position, String dataType, BytesR private static Object valueAt(String dataType, Block block, int offset, BytesRef scratch) { return switch (dataType) { case "unsigned_long" -> unsignedLongAsNumber(((LongBlock) block).getLong(offset)); - case "long" -> ((LongBlock) block).getLong(offset); - case "integer" -> ((IntBlock) block).getInt(offset); - case "double" -> ((DoubleBlock) block).getDouble(offset); + case "long", "counter_long" -> ((LongBlock) block).getLong(offset); + case "integer", "counter_integer" -> ((IntBlock) block).getInt(offset); + case "double", "counter_double" -> ((DoubleBlock) block).getDouble(offset); case "keyword", "text" -> ((BytesRefBlock) block).getBytesRef(offset, scratch).utf8ToString(); case "ip" -> { BytesRef val = ((BytesRefBlock) block).getBytesRef(offset, scratch); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java index 74cf0c4c1deea..8e837dde9e387 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.InvalidArgumentException; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.NodeInfo; @@ -42,7 +43,8 @@ public class ToDouble extends AbstractConvertFunction { Map.entry(TEXT, ToDoubleFromStringEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToDoubleFromUnsignedLongEvaluator.Factory::new), Map.entry(LONG, ToDoubleFromLongEvaluator.Factory::new), // CastLongToDoubleEvaluator would be a candidate, but not MV'd - Map.entry(INTEGER, ToDoubleFromIntEvaluator.Factory::new) // CastIntToDoubleEvaluator would be a candidate, but not MV'd + Map.entry(INTEGER, ToDoubleFromIntEvaluator.Factory::new), // CastIntToDoubleEvaluator would be a candidate, but not MV'd + Map.entry(EsqlDataTypes.COUNTER_DOUBLE, (field, source) -> field) ); @FunctionInfo( @@ -65,7 +67,7 @@ public ToDouble( Source source, @Param( name = "field", - type = { "boolean", "date", "keyword", "text", "double", "long", "unsigned_long", "integer" }, + type = { "boolean", "date", "keyword", "text", "double", "long", "unsigned_long", "integer", "counter_double" }, description = "Input value. The input can be a single- or multi-valued column or an expression." ) Expression field ) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java index d50f1f613b589..32e3b8a77695c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToInteger.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.InvalidArgumentException; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.NodeInfo; @@ -43,7 +44,8 @@ public class ToInteger extends AbstractConvertFunction { Map.entry(TEXT, ToIntegerFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToIntegerFromDoubleEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToIntegerFromUnsignedLongEvaluator.Factory::new), - Map.entry(LONG, ToIntegerFromLongEvaluator.Factory::new) + Map.entry(LONG, ToIntegerFromLongEvaluator.Factory::new), + Map.entry(EsqlDataTypes.COUNTER_INTEGER, (fieldEval, source) -> fieldEval) ); @FunctionInfo( @@ -68,7 +70,7 @@ public ToInteger( Source source, @Param( name = "field", - type = { "boolean", "date", "keyword", "text", "double", "long", "unsigned_long", "integer" }, + type = { "boolean", "date", "keyword", "text", "double", "long", "unsigned_long", "integer", "counter_integer" }, description = "Input value. The input can be a single- or multi-valued column or an expression." ) Expression field ) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java index 77973ec49b7e3..9b8f01542e80a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.InvalidArgumentException; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.NodeInfo; @@ -43,7 +44,8 @@ public class ToLong extends AbstractConvertFunction { Map.entry(TEXT, ToLongFromStringEvaluator.Factory::new), Map.entry(DOUBLE, ToLongFromDoubleEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToLongFromUnsignedLongEvaluator.Factory::new), - Map.entry(INTEGER, ToLongFromIntEvaluator.Factory::new) // CastIntToLongEvaluator would be a candidate, but not MV'd + Map.entry(INTEGER, ToLongFromIntEvaluator.Factory::new), // CastIntToLongEvaluator would be a candidate, but not MV'd + Map.entry(EsqlDataTypes.COUNTER_LONG, (field, source) -> field) ); @FunctionInfo( @@ -67,7 +69,7 @@ public ToLong( Source source, @Param( name = "field", - type = { "boolean", "date", "keyword", "text", "double", "long", "unsigned_long", "integer" }, + type = { "boolean", "date", "keyword", "text", "double", "long", "unsigned_long", "integer", "counter_long" }, description = "Input value. The input can be a single- or multi-valued column or an expression." ) Expression field ) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index 3ea3bd54da135..b61eba5eab904 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -343,7 +343,8 @@ private PhysicalOperation planTopN(TopNExec topNExec, LocalExecutionPlannerConte case "text", "keyword" -> TopNEncoder.UTF8; case "version" -> TopNEncoder.VERSION; case "boolean", "null", "byte", "short", "integer", "long", "double", "float", "half_float", "datetime", "date_period", - "time_duration", "object", "nested", "scaled_float", "unsigned_long", "_doc" -> TopNEncoder.DEFAULT_SORTABLE; + "time_duration", "object", "nested", "scaled_float", "unsigned_long", "_doc", "counter_integer", "counter_long", + "counter_double" -> TopNEncoder.DEFAULT_SORTABLE; case "geo_point", "cartesian_point", "geo_shape", "cartesian_shape" -> TopNEncoder.DEFAULT_UNSORTABLE; // unsupported fields are encoded as BytesRef, we'll use the same encoder; all values should be null at this point case "unsupported" -> TopNEncoder.UNSUPPORTED; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index 98bf932ce3af8..26c57f13e16c4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -251,13 +251,16 @@ public static ElementType toElementType(DataType dataType) { * For example, spatial types can be extracted into doc-values under specific conditions, otherwise they extract as BytesRef. */ public static ElementType toElementType(DataType dataType, MappedFieldType.FieldExtractPreference fieldExtractPreference) { - if (dataType == DataTypes.LONG || dataType == DataTypes.DATETIME || dataType == DataTypes.UNSIGNED_LONG) { + if (dataType == DataTypes.LONG + || dataType == DataTypes.DATETIME + || dataType == DataTypes.UNSIGNED_LONG + || dataType == EsqlDataTypes.COUNTER_LONG) { return ElementType.LONG; } - if (dataType == DataTypes.INTEGER) { + if (dataType == DataTypes.INTEGER || dataType == EsqlDataTypes.COUNTER_INTEGER) { return ElementType.INT; } - if (dataType == DataTypes.DOUBLE) { + if (dataType == DataTypes.DOUBLE || dataType == EsqlDataTypes.COUNTER_DOUBLE) { return ElementType.DOUBLE; } // unsupported fields are passed through as a BytesRef diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java index 89c7455baf885..b508e9a4f040c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java @@ -121,6 +121,11 @@ public class EsqlFeatures implements FeatureSpecification { */ public static final NodeFeature MV_ORDERING_SORTED_ASCENDING = new NodeFeature("esql.mv_ordering_sorted_ascending"); + /** + * Support for metrics counter fields + */ + public static final NodeFeature METRICS_COUNTER_FIELDS = new NodeFeature("esql.metrics_counter_fields"); + @Override public Set getFeatures() { return Set.of( @@ -139,7 +144,8 @@ public Set getFeatures() { ST_DISJOINT, STRING_LITERAL_AUTO_CASTING, CASTING_OPERATOR, - MV_ORDERING_SORTED_ASCENDING + MV_ORDERING_SORTED_ASCENDING, + METRICS_COUNTER_FIELDS ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java index 2910a690bf8a0..e763d54a2dcf4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java @@ -10,7 +10,6 @@ import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypeRegistry; -import org.elasticsearch.xpack.ql.type.DataTypes; import java.util.Collection; @@ -37,10 +36,10 @@ public Collection dataTypes() { @Override public DataType fromEs(String typeName, TimeSeriesParams.MetricType metricType) { if (metricType == TimeSeriesParams.MetricType.COUNTER) { - // Counter fields will be a counter type, for now they are unsupported - return DataTypes.UNSUPPORTED; + return EsqlDataTypes.getCounterType(typeName); + } else { + return EsqlDataTypes.fromName(typeName); } - return EsqlDataTypes.fromName(typeName); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java index 468ffcc2cba2a..0725b270b858a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java @@ -51,6 +51,10 @@ public final class EsqlDataTypes { public static final DataType GEO_SHAPE = new DataType("geo_shape", Integer.MAX_VALUE, false, false, true); public static final DataType CARTESIAN_SHAPE = new DataType("cartesian_shape", Integer.MAX_VALUE, false, false, true); + public static final DataType COUNTER_LONG = new DataType("counter_long", LONG.size(), false, false, true); + public static final DataType COUNTER_INTEGER = new DataType("counter_integer", INTEGER.size(), false, false, true); + public static final DataType COUNTER_DOUBLE = new DataType("counter_double", DOUBLE.size(), false, false, true); + private static final Collection TYPES = Stream.of( BOOLEAN, UNSUPPORTED, @@ -77,7 +81,10 @@ public final class EsqlDataTypes { GEO_POINT, CARTESIAN_POINT, CARTESIAN_SHAPE, - GEO_SHAPE + GEO_SHAPE, + COUNTER_LONG, + COUNTER_INTEGER, + COUNTER_DOUBLE ).sorted(Comparator.comparing(DataType::typeName)).toList(); private static final Map NAME_TO_TYPE = TYPES.stream().collect(toUnmodifiableMap(DataType::typeName, t -> t)); @@ -212,7 +219,8 @@ public static boolean isRepresentable(DataType t) { && t != FLOAT && t != SCALED_FLOAT && t != SOURCE - && t != HALF_FLOAT; + && t != HALF_FLOAT + && isCounterType(t) == false; } public static boolean areCompatible(DataType left, DataType right) { @@ -232,4 +240,12 @@ public static DataType widenSmallNumericTypes(DataType type) { } return type; } + + public static DataType getCounterType(String typeName) { + return fromTypeName("counter_" + typeName); + } + + public static boolean isCounterType(DataType dt) { + return dt == COUNTER_LONG || dt == COUNTER_INTEGER || dt == COUNTER_DOUBLE; + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index 0e2886d099916..f0aad51789b5a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -125,7 +125,10 @@ EsqlQueryResponse randomResponseAsync(boolean columnar, EsqlQueryResponse.Profil private ColumnInfo randomColumnInfo() { DataType type = randomValueOtherThanMany( - t -> false == DataTypes.isPrimitive(t) || t == EsqlDataTypes.DATE_PERIOD || t == EsqlDataTypes.TIME_DURATION, + t -> false == DataTypes.isPrimitive(t) + || t == EsqlDataTypes.DATE_PERIOD + || t == EsqlDataTypes.TIME_DURATION + || EsqlDataTypes.isCounterType(t), () -> randomFrom(EsqlDataTypes.types()) ); type = EsqlDataTypes.widenSmallNumericTypes(type); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index 3e1fbaa2940eb..32454333d4621 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -121,11 +121,11 @@ public static Literal randomLiteral(DataType type) { case "boolean" -> randomBoolean(); case "byte" -> randomByte(); case "short" -> randomShort(); - case "integer" -> randomInt(); - case "unsigned_long", "long" -> randomLong(); + case "integer", "counter_integer" -> randomInt(); + case "unsigned_long", "long", "counter_long" -> randomLong(); case "date_period" -> Period.of(randomIntBetween(-1000, 1000), randomIntBetween(-13, 13), randomIntBetween(-32, 32)); case "datetime" -> randomMillisUpToYear9999(); - case "double", "scaled_float" -> randomDouble(); + case "double", "scaled_float", "counter_double" -> randomDouble(); case "float" -> randomFloat(); case "half_float" -> HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(randomFloat())); case "keyword" -> new BytesRef(randomAlphaOfLength(5)); @@ -946,6 +946,52 @@ protected static String typeErrorMessage(boolean includeOrdinal, List parameters() { List.of() ); + TestCaseSupplier.unary( + suppliers, + "noop", + List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomDouble, EsqlDataTypes.COUNTER_DOUBLE)), + DataTypes.DOUBLE, + l -> l, + List.of() + ); + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java index e6f6cb7e978f7..b35915bfbb038 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; @@ -257,6 +258,15 @@ public static Iterable parameters() { ) ); + TestCaseSupplier.unary( + suppliers, + "noop", + List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomInt, EsqlDataTypes.COUNTER_INTEGER)), + DataTypes.INTEGER, + l -> l, + List.of() + ); + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java index 1879b7ce97ea8..9da7e377ab3eb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java @@ -11,8 +11,10 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; @@ -208,6 +210,14 @@ public static Iterable parameters() { ) ); + TestCaseSupplier.unary( + suppliers, + "noop", + List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomNonNegativeLong, EsqlDataTypes.COUNTER_LONG)), + DataTypes.LONG, + l -> l, + List.of() + ); return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java index 93f58398d267f..23d2f8da488e1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java @@ -23,8 +23,12 @@ import static org.hamcrest.Matchers.equalTo; public class EsqlDataTypeRegistryTests extends ESTestCase { + public void testCounter() { - resolve("long", TimeSeriesParams.MetricType.COUNTER, DataTypes.UNSUPPORTED); + resolve("long", TimeSeriesParams.MetricType.COUNTER, EsqlDataTypes.COUNTER_LONG); + resolve("integer", TimeSeriesParams.MetricType.COUNTER, EsqlDataTypes.COUNTER_INTEGER); + resolve("double", TimeSeriesParams.MetricType.COUNTER, EsqlDataTypes.COUNTER_DOUBLE); + } public void testGauge() { diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml index 30b81860f014f..6e6e54fcb1d13 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml @@ -1,7 +1,7 @@ setup: - requires: - cluster_features: ["gte_v8.11.0"] - reason: "ESQL is available in 8.11+" + cluster_features: ["esql.metrics_counter_fields"] + reason: "require metrics counter fields" test_runner_features: allowed_warnings_regex - do: indices.create: @@ -38,7 +38,7 @@ setup: type: long time_series_metric: counter rx: - type: long + type: integer time_series_metric: counter - do: bulk: @@ -112,7 +112,6 @@ load everything: reason: "_source is available in 8.13+" - do: allowed_warnings_regex: - - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: @@ -126,9 +125,9 @@ load everything: - match: {columns.2.name: "k8s.pod.name"} - match: {columns.2.type: "keyword"} - match: {columns.3.name: "k8s.pod.network.rx"} - - match: {columns.3.type: "unsupported"} + - match: {columns.3.type: "counter_integer"} - match: {columns.4.name: "k8s.pod.network.tx"} - - match: {columns.4.type: "unsupported"} + - match: {columns.4.type: "counter_long"} - match: {columns.5.name: "k8s.pod.uid"} - match: {columns.5.type: "keyword"} - match: {columns.6.name: "metricset"} @@ -139,7 +138,6 @@ load everything: load a document: - do: allowed_warnings_regex: - - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: @@ -151,25 +149,41 @@ load a document: - match: {values.0.0: "2021-04-28T18:50:23.142Z"} - match: {values.0.1: "10.10.55.3"} - match: {values.0.2: "dog"} - - match: {values.0.3: null } - - match: {values.0.4: null } + - match: {values.0.3: 530600088 } + - match: {values.0.4: 1434577921 } - match: {values.0.5: "df3145b3-0563-4d3b-a0f7-897eb2876ea9"} - match: {values.0.6: "pod"} --- filter on counter: - do: - catch: /Cannot use field \[k8s.pod.network.tx\] with unsupported type \[counter\]/ + catch: bad_request esql.query: body: query: 'from test | where k8s.pod.network.tx == 1434577921' version: 2024.04.01 +--- +cast counter then filter: + - do: + esql.query: + body: + query: 'from test | where k8s.pod.network.tx::long == 2005177954 and k8s.pod.network.rx::integer == 801479970 | sort @timestamp | limit 10' + version: 2024.04.01 + - length: {values: 1} + - length: {values.0: 7} + - match: {values.0.0: "2021-04-28T18:50:24.467Z"} + - match: {values.0.1: "10.10.55.1"} + - match: {values.0.2: "cat"} + - match: {values.0.3: 801479970 } + - match: {values.0.4: 2005177954 } + - match: {values.0.5: "947e4ced-1786-4e53-9e0c-5c447e959507"} + - match: {values.0.6: "pod"} + --- from doc with aggregate_metric_double: - do: allowed_warnings_regex: - - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: @@ -201,7 +215,6 @@ stats on aggregate_metric_double: from index pattern unsupported counter: - do: allowed_warnings_regex: - - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: @@ -219,7 +232,7 @@ from index pattern unsupported counter: - match: {columns.4.name: "k8s.pod.name"} - match: {columns.4.type: "keyword"} - match: {columns.5.name: "k8s.pod.network.rx"} - - match: {columns.5.type: "unsupported"} + - match: {columns.5.type: "counter_integer"} - match: {columns.6.name: "k8s.pod.network.tx"} - match: {columns.6.type: "unsupported"} - match: {columns.7.name: "k8s.pod.uid"} From d3aff8dcffba612ff22920aca0eaead519c722cc Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Wed, 24 Apr 2024 23:25:05 -0700 Subject: [PATCH 02/13] Update docs/changelog/107877.yaml --- docs/changelog/107877.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/107877.yaml diff --git a/docs/changelog/107877.yaml b/docs/changelog/107877.yaml new file mode 100644 index 0000000000000..ccbdbb6939906 --- /dev/null +++ b/docs/changelog/107877.yaml @@ -0,0 +1,5 @@ +pr: 107877 +summary: Support metrics counter types in ESQL +area: "ES|QL, TSDB" +type: enhancement +issues: [] From 06b30783b80286edfbf80f48619ec7e0fca0e714 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 25 Apr 2024 08:16:53 -0700 Subject: [PATCH 03/13] fix topn --- .../esql/planner/LocalExecutionPlanner.java | 6 +++--- .../rest-api-spec/test/esql/40_tsdb.yml | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index b61eba5eab904..e7285bae32408 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -343,9 +343,9 @@ private PhysicalOperation planTopN(TopNExec topNExec, LocalExecutionPlannerConte case "text", "keyword" -> TopNEncoder.UTF8; case "version" -> TopNEncoder.VERSION; case "boolean", "null", "byte", "short", "integer", "long", "double", "float", "half_float", "datetime", "date_period", - "time_duration", "object", "nested", "scaled_float", "unsigned_long", "_doc", "counter_integer", "counter_long", - "counter_double" -> TopNEncoder.DEFAULT_SORTABLE; - case "geo_point", "cartesian_point", "geo_shape", "cartesian_shape" -> TopNEncoder.DEFAULT_UNSORTABLE; + "time_duration", "object", "nested", "scaled_float", "unsigned_long", "_doc" -> TopNEncoder.DEFAULT_SORTABLE; + case "geo_point", "cartesian_point", "geo_shape", "cartesian_shape", "counter_long", "counter_integer", "counter_double" -> + TopNEncoder.DEFAULT_UNSORTABLE; // unsupported fields are encoded as BytesRef, we'll use the same encoder; all values should be null at this point case "unsupported" -> TopNEncoder.UNSUPPORTED; default -> throw new EsqlIllegalArgumentException("No TopN sorting encoder for type " + inverse.get(channel).type()); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml index 6e6e54fcb1d13..400c37f1e4a00 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml @@ -180,6 +180,25 @@ cast counter then filter: - match: {values.0.5: "947e4ced-1786-4e53-9e0c-5c447e959507"} - match: {values.0.6: "pod"} +--- +sort on counter without cast: + - do: + catch: bad_request + esql.query: + body: + query: 'from test | KEEP k8s.pod.network.tx | sort @k8s.pod.network.tx | limit 1' + version: 2024.04.01 + +--- +cast then sort on counter: + - do: + esql.query: + body: + query: 'from test | KEEP k8s.pod.network.tx | EVAL tx=to_long(k8s.pod.network.tx) | sort tx | limit 1' + version: 2024.04.01 + - length: {values: 1} + - match: {values.0.0: 1434521831 } + --- from doc with aggregate_metric_double: - do: From 0222b149aa4078f95d7c898a09813c84d23b6732 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 25 Apr 2024 08:17:01 -0700 Subject: [PATCH 04/13] rename --- .../yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml index 400c37f1e4a00..c09bc17ab9a5c 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml @@ -155,7 +155,7 @@ load a document: - match: {values.0.6: "pod"} --- -filter on counter: +filter on counter without cast: - do: catch: bad_request esql.query: From aa3e42e23a9a9e04c1c4375fc2575cbf275cc441 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 25 Apr 2024 09:06:11 -0700 Subject: [PATCH 05/13] more tests --- .../esql/expression/function/scalar/convert/ToDoubleTests.java | 2 +- .../esql/expression/function/scalar/convert/ToIntegerTests.java | 2 +- .../esql/expression/function/scalar/convert/ToLongTests.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java index 527de53a4a605..15445220eecf7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java @@ -117,7 +117,7 @@ public static Iterable parameters() { TestCaseSupplier.unary( suppliers, - "noop", + "Attribute[channel=0]", List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomDouble, EsqlDataTypes.COUNTER_DOUBLE)), DataTypes.DOUBLE, l -> l, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java index b35915bfbb038..bc27ded5a6dae 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java @@ -260,7 +260,7 @@ public static Iterable parameters() { TestCaseSupplier.unary( suppliers, - "noop", + "Attribute[channel=0]", List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomInt, EsqlDataTypes.COUNTER_INTEGER)), DataTypes.INTEGER, l -> l, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java index 9da7e377ab3eb..cd515d4afd3c9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java @@ -212,7 +212,7 @@ public static Iterable parameters() { TestCaseSupplier.unary( suppliers, - "noop", + "Attribute[channel=0]", List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomNonNegativeLong, EsqlDataTypes.COUNTER_LONG)), DataTypes.LONG, l -> l, From 5c2b08fbbb180065b34ae4ea50cb377fafe16712 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 25 Apr 2024 09:09:02 -0700 Subject: [PATCH 06/13] response --- .../xpack/esql/action/ResponseValueUtils.java | 6 +++--- .../xpack/esql/action/EsqlQueryResponseTests.java | 11 ++++------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java index 97b171fc0f070..ba9aafe03143f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java @@ -174,9 +174,9 @@ static Page valuesToPage(BlockFactory blockFactory, List columns, Li case "unsigned_long" -> ((LongBlock.Builder) builder).appendLong( longToUnsignedLong(((Number) value).longValue(), true) ); - case "long" -> ((LongBlock.Builder) builder).appendLong(((Number) value).longValue()); - case "integer" -> ((IntBlock.Builder) builder).appendInt(((Number) value).intValue()); - case "double" -> ((DoubleBlock.Builder) builder).appendDouble(((Number) value).doubleValue()); + case "long", "counter_long" -> ((LongBlock.Builder) builder).appendLong(((Number) value).longValue()); + case "integer", "counter_integer" -> ((IntBlock.Builder) builder).appendInt(((Number) value).intValue()); + case "double", "counter_double" -> ((DoubleBlock.Builder) builder).appendDouble(((Number) value).doubleValue()); case "keyword", "text", "unsupported" -> ((BytesRefBlock.Builder) builder).appendBytesRef( new BytesRef(value.toString()) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index f0aad51789b5a..79939365181aa 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -125,10 +125,7 @@ EsqlQueryResponse randomResponseAsync(boolean columnar, EsqlQueryResponse.Profil private ColumnInfo randomColumnInfo() { DataType type = randomValueOtherThanMany( - t -> false == DataTypes.isPrimitive(t) - || t == EsqlDataTypes.DATE_PERIOD - || t == EsqlDataTypes.TIME_DURATION - || EsqlDataTypes.isCounterType(t), + t -> false == DataTypes.isPrimitive(t) || t == EsqlDataTypes.DATE_PERIOD || t == EsqlDataTypes.TIME_DURATION, () -> randomFrom(EsqlDataTypes.types()) ); type = EsqlDataTypes.widenSmallNumericTypes(type); @@ -146,9 +143,9 @@ private Page randomPage(List columns) { return new Page(columns.stream().map(c -> { Block.Builder builder = PlannerUtils.toElementType(EsqlDataTypes.fromName(c.type())).newBlockBuilder(1, blockFactory); switch (c.type()) { - case "unsigned_long", "long" -> ((LongBlock.Builder) builder).appendLong(randomLong()); - case "integer" -> ((IntBlock.Builder) builder).appendInt(randomInt()); - case "double" -> ((DoubleBlock.Builder) builder).appendDouble(randomDouble()); + case "unsigned_long", "long", "counter_long" -> ((LongBlock.Builder) builder).appendLong(randomLong()); + case "integer", "counter_integer" -> ((IntBlock.Builder) builder).appendInt(randomInt()); + case "double", "counter_double" -> ((DoubleBlock.Builder) builder).appendDouble(randomDouble()); case "keyword" -> ((BytesRefBlock.Builder) builder).appendBytesRef(new BytesRef(randomAlphaOfLength(10))); case "text" -> ((BytesRefBlock.Builder) builder).appendBytesRef(new BytesRef(randomAlphaOfLength(10000))); case "ip" -> ((BytesRefBlock.Builder) builder).appendBytesRef( From 8ec3e66bc90e1041c3a746171a31676f48f888da Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 25 Apr 2024 09:10:05 -0700 Subject: [PATCH 07/13] changelog --- docs/changelog/107877.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/107877.yaml b/docs/changelog/107877.yaml index ccbdbb6939906..cf458b3aa3a42 100644 --- a/docs/changelog/107877.yaml +++ b/docs/changelog/107877.yaml @@ -1,5 +1,5 @@ pr: 107877 summary: Support metrics counter types in ESQL -area: "ES|QL, TSDB" +area: "ES|QL" type: enhancement issues: [] From 472fbd2de7b86babfbf5076395b0519ac7687595 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 25 Apr 2024 09:20:59 -0700 Subject: [PATCH 08/13] javadoc --- .../xpack/esql/type/EsqlDataTypes.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java index 0725b270b858a..912c17dae0865 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java @@ -51,9 +51,16 @@ public final class EsqlDataTypes { public static final DataType GEO_SHAPE = new DataType("geo_shape", Integer.MAX_VALUE, false, false, true); public static final DataType CARTESIAN_SHAPE = new DataType("cartesian_shape", Integer.MAX_VALUE, false, false, true); - public static final DataType COUNTER_LONG = new DataType("counter_long", LONG.size(), false, false, true); - public static final DataType COUNTER_INTEGER = new DataType("counter_integer", INTEGER.size(), false, false, true); - public static final DataType COUNTER_DOUBLE = new DataType("counter_double", DOUBLE.size(), false, false, true); + /** + * These are numeric fields labeled as metric counters in time-series indices. Although stored + * internally as numeric fields, they represent cumulative metrics and must not be treated as regular + * numeric fields. Therefore, we define them differently and separately from their parent numeric field. + * These fields are strictly for use in retrieval from indices, rate aggregation, and casting to their + * parent numeric type. + */ + public static final DataType COUNTER_LONG = new DataType("counter_long", LONG.size(), false, false, LONG.hasDocValues()); + public static final DataType COUNTER_INTEGER = new DataType("counter_integer", INTEGER.size(), false, false, INTEGER.hasDocValues()); + public static final DataType COUNTER_DOUBLE = new DataType("counter_double", DOUBLE.size(), false, false, DOUBLE.hasDocValues()); private static final Collection TYPES = Stream.of( BOOLEAN, From 5b09bdfeaed3749f920b1c1b050ccbe9a3482136 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 25 Apr 2024 10:56:59 -0700 Subject: [PATCH 09/13] strict validation --- .../src/main/resources/tsdb-mapping.json | 30 ++++++++++++++ .../xpack/esql/analysis/Verifier.java | 4 ++ .../expression/function/aggregate/Count.java | 11 ++++- .../function/aggregate/NumericAggregate.java | 4 +- .../esql/analysis/AnalyzerTestUtils.java | 4 ++ .../xpack/esql/analysis/AnalyzerTests.java | 41 ++++++++++--------- .../xpack/esql/analysis/VerifierTests.java | 40 +++++++++++++++++- .../elasticsearch/xpack/ql/type/Types.java | 8 +++- .../xpack/ql/type/TypesTests.java | 3 +- 9 files changed, 118 insertions(+), 27 deletions(-) create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/tsdb-mapping.json diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/tsdb-mapping.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/tsdb-mapping.json new file mode 100644 index 0000000000000..c3bba9724602b --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/tsdb-mapping.json @@ -0,0 +1,30 @@ +{ + "properties": { + "@timestamp": { + "type": "date" + }, + "metricset": { + "type": "keyword", + "time_series_dimension": true + }, + "name": { + "type": "keyword" + }, + "network": { + "properties": { + "connections": { + "type": "long", + "time_series_metric": "gauge" + }, + "bytes_in": { + "type": "long", + "time_series_metric": "counter" + }, + "bytes_out": { + "type": "long", + "time_series_metric": "counter" + } + } + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 2267125304da7..b318e7ed99bc0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.ql.expression.AttributeSet; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Expressions; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.NamedExpression; import org.elasticsearch.xpack.ql.expression.TypeResolutions; import org.elasticsearch.xpack.ql.expression.function.aggregate.AggregateFunction; @@ -193,6 +194,9 @@ private static void checkAggregate(LogicalPlan p, Set failures) { if (attr != null) { groupRefs.add(attr); } + if (e instanceof FieldAttribute f && EsqlDataTypes.isCounterType(f.dataType())) { + failures.add(fail(e, "cannot group by on [{}] type for grouping [{}]", f.dataType().typeName(), e.sourceText())); + } }); // check aggregates - accept only aggregate functions or expressions over grouping diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java index 7ce655bf59962..6aa4bd70e42c6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java @@ -9,7 +9,6 @@ import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.CountAggregatorFunction; -import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; import org.elasticsearch.xpack.esql.expression.SurrogateExpression; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; @@ -17,6 +16,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mul; import org.elasticsearch.xpack.esql.planner.ToAggregator; +import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.expression.Nullability; @@ -31,6 +31,7 @@ import java.util.List; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isType; public class Count extends AggregateFunction implements EnclosedAgg, ToAggregator, SurrogateExpression { @@ -91,7 +92,13 @@ public Nullability nullable() { @Override protected TypeResolution resolveType() { - return EsqlTypeResolutions.isExact(field(), sourceText(), DEFAULT); + return isType( + field(), + dt -> EsqlDataTypes.isCounterType(dt) == false, + sourceText(), + DEFAULT, + "any type except counter_long or counter_integer or counter_double" + ); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java index 8e1e38441e9a6..cd86f51bb1fd6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java @@ -40,7 +40,7 @@ protected TypeResolution resolveType() { sourceText(), DEFAULT, "datetime", - "numeric except unsigned_long" + "numeric except unsigned_long or counter_long or counter_integer or counter_double" ); } return isType( @@ -48,7 +48,7 @@ protected TypeResolution resolveType() { dt -> dt.isNumeric() && dt != DataTypes.UNSIGNED_LONG, sourceText(), DEFAULT, - "numeric except unsigned_long" + "numeric except unsigned_long or counter_long or counter_integer or counter_double" ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java index ad8cb1003eeaa..a94cba52f8f0a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java @@ -134,4 +134,8 @@ public static void loadEnrichPolicyResolution( public static void loadEnrichPolicyResolution(EnrichResolution enrich, String policy, String field, String index, String mapping) { loadEnrichPolicyResolution(enrich, EnrichPolicy.MATCH_TYPE, policy, field, index, mapping); } + + public static IndexResolution tsdbIndexResolution() { + return loadMapping("tsdb-mapping.json", "test"); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 8f474e6cb6a83..4c158de233865 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -1784,18 +1784,19 @@ public void testUnsupportedTypesInStats() { """, "Found 8 problems\n" + "line 2:12: argument of [avg(x)] must be [numeric except unsigned_long], found value [x] type [unsigned_long]\n" - + "line 2:20: argument of [count_distinct(x)] must be [any exact type except unsigned_long], " - + "found value [x] type [unsigned_long]\n" - + "line 2:39: argument of [max(x)] must be [datetime or numeric except unsigned_long], " - + "found value [max(x)] type [unsigned_long]\n" + + "line 2:20: argument of [count_distinct(x)] must be [any exact type except unsigned_long], found value [x] " + + "type [unsigned_long]\n" + + "line 2:39: argument of [max(x)] must be [datetime or numeric except unsigned_long or counter_long or " + + "counter_integer or counter_double], found value [max(x)] type [unsigned_long]\n" + "line 2:47: argument of [median(x)] must be [numeric except unsigned_long], found value [x] type [unsigned_long]\n" - + "line 2:58: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long], " - + "found value [x] type [unsigned_long]\n" - + "line 2:88: argument of [min(x)] must be [datetime or numeric except unsigned_long], " - + "found value [min(x)] type [unsigned_long]\n" - + "line 2:96: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], " - + "found value [x] type [unsigned_long]\n" - + "line 2:115: argument of [sum(x)] must be [numeric except unsigned_long], found value [x] type [unsigned_long]" + + "line 2:58: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter_long or " + + "counter_integer or counter_double], found value [x] type [unsigned_long]\n" + + "line 2:88: argument of [min(x)] must be [datetime or numeric except unsigned_long or counter_long or counter_integer" + + " or counter_double], found value [min(x)] type [unsigned_long]\n" + + "line 2:96: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], " + + "found value [x] type [unsigned_long]\n" + + "line 2:115: argument of [sum(x)] must be [numeric except unsigned_long or counter_long or counter_integer " + + "or counter_double], found value [x] type [unsigned_long]" ); verifyUnsupported( @@ -1805,16 +1806,16 @@ public void testUnsupportedTypesInStats() { """, "Found 7 problems\n" + "line 2:10: argument of [avg(x)] must be [numeric except unsigned_long], found value [x] type [version]\n" - + "line 2:18: argument of [max(x)] must be [datetime or numeric except unsigned_long], " - + "found value [max(x)] type [version]\n" + + "line 2:18: argument of [max(x)] must be [datetime or numeric except unsigned_long or counter_long or " + + "counter_integer or counter_double], found value [max(x)] type [version]\n" + "line 2:26: argument of [median(x)] must be [numeric except unsigned_long], found value [x] type [version]\n" - + "line 2:37: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long], " - + "found value [x] type [version]\n" - + "line 2:67: argument of [min(x)] must be [datetime or numeric except unsigned_long], " - + "found value [min(x)] type [version]\n" - + "line 2:75: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], " - + "found value [x] type [version]\n" - + "line 2:94: argument of [sum(x)] must be [numeric except unsigned_long], found value [x] type [version]" + + "line 2:37: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter_long " + + "or counter_integer or counter_double], found value [x] type [version]\n" + + "line 2:67: argument of [min(x)] must be [datetime or numeric except unsigned_long or counter_long or counter_integer " + + "or counter_double], found value [min(x)] type [version]\n" + + "line 2:75: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], found value [x] type [version]\n" + + "line 2:94: argument of [sum(x)] must be [numeric except unsigned_long or counter_long or counter_integer or " + + "counter_double], found value [x] type [version]" ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 8d9140cdda5f4..193c9cea6dc5e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -21,12 +21,14 @@ import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; //@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE,org.elasticsearch.compute:TRACE", reason = "debug") public class VerifierTests extends ESTestCase { private static final EsqlParser parser = new EsqlParser(); private final Analyzer defaultAnalyzer = AnalyzerTestUtils.expandedDefaultAnalyzer(); + private final Analyzer tsdb = AnalyzerTestUtils.analyzer(AnalyzerTestUtils.tsdbIndexResolution()); public void testIncompatibleTypesInMathOperation() { assertEquals( @@ -378,7 +380,8 @@ public void testUnsignedLongNegation() { public void testSumOnDate() { assertEquals( - "1:19: argument of [sum(hire_date)] must be [numeric except unsigned_long], found value [hire_date] type [datetime]", + "1:19: argument of [sum(hire_date)] must be [numeric except unsigned_long or counter_long or" + + " counter_integer or counter_double], found value [hire_date] type [datetime]", error("from test | stats sum(hire_date)") ); } @@ -480,6 +483,41 @@ public void testInlineImpossibleConvert() { assertEquals("1:5: argument of [false::ip] must be [ip or string], found value [false] type [boolean]", error("ROW false::ip")); } + public void testAggregateOnCounter() { + assertThat( + error("FROM tests | STATS min(network.bytes_in)", tsdb), + equalTo( + "1:20: argument of [min(network.bytes_in)] must be " + + "[datetime or numeric except unsigned_long or counter_long or counter_integer or counter_double], " + + "found value [min(network.bytes_in)] type [counter_long]" + ) + ); + + assertThat( + error("FROM tests | STATS max(network.bytes_in)", tsdb), + equalTo( + "1:20: argument of [max(network.bytes_in)] must be " + + "[datetime or numeric except unsigned_long or counter_long or counter_integer or counter_double], " + + "found value [max(network.bytes_in)] type [counter_long]" + ) + ); + + assertThat( + error("FROM tests | STATS count(network.bytes_out)", tsdb), + equalTo( + "1:20: argument of [count(network.bytes_out)] must be [any type except counter_long or counter_integer" + + " or counter_double], found value [network.bytes_out] type [counter_long]" + ) + ); + } + + public void testGroupByCounter() { + assertThat( + error("FROM tests | STATS count(*) BY network.bytes_in", tsdb), + equalTo("1:32: cannot group by on [counter_long] type for grouping [network.bytes_in]") + ); + } + private String error(String query) { return error(query, defaultAnalyzer); } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java index a19f4c634f77c..00f776db29fb6 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java @@ -53,7 +53,13 @@ private static DataType getType(DataTypeRegistry typeRegistry, Map loadMapping(DataTypeRegistry registry, String private static Map loadMapping(DataTypeRegistry registry, InputStream stream, Boolean ordered) { boolean order = ordered != null ? ordered.booleanValue() : randomBoolean(); try (InputStream in = stream) { - return Types.fromEs(registry, XContentHelper.convertToMap(JsonXContent.jsonXContent, in, order)); + Map map = XContentHelper.convertToMap(JsonXContent.jsonXContent, in, order); + return Types.fromEs(registry, map); } catch (IOException ex) { throw new RuntimeException(ex); } From 8491d321a050af69b41b3fad4bd13b5463bf9c7b Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 25 Apr 2024 11:03:22 -0700 Subject: [PATCH 10/13] stylecheck --- .../xpack/esql/analysis/AnalyzerTests.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 4c158de233865..4af28b6b41f93 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -1784,19 +1784,19 @@ public void testUnsupportedTypesInStats() { """, "Found 8 problems\n" + "line 2:12: argument of [avg(x)] must be [numeric except unsigned_long], found value [x] type [unsigned_long]\n" - + "line 2:20: argument of [count_distinct(x)] must be [any exact type except unsigned_long], found value [x] " + - "type [unsigned_long]\n" - + "line 2:39: argument of [max(x)] must be [datetime or numeric except unsigned_long or counter_long or " + - "counter_integer or counter_double], found value [max(x)] type [unsigned_long]\n" + + "line 2:20: argument of [count_distinct(x)] must be [any exact type except unsigned_long], found value [x] " + + "type [unsigned_long]\n" + + "line 2:39: argument of [max(x)] must be [datetime or numeric except unsigned_long or counter_long or " + + "counter_integer or counter_double], found value [max(x)] type [unsigned_long]\n" + "line 2:47: argument of [median(x)] must be [numeric except unsigned_long], found value [x] type [unsigned_long]\n" - + "line 2:58: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter_long or " + - "counter_integer or counter_double], found value [x] type [unsigned_long]\n" - + "line 2:88: argument of [min(x)] must be [datetime or numeric except unsigned_long or counter_long or counter_integer" + - " or counter_double], found value [min(x)] type [unsigned_long]\n" - + "line 2:96: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], " + - "found value [x] type [unsigned_long]\n" - + "line 2:115: argument of [sum(x)] must be [numeric except unsigned_long or counter_long or counter_integer " + - "or counter_double], found value [x] type [unsigned_long]" + + "line 2:58: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter_long or " + + "counter_integer or counter_double], found value [x] type [unsigned_long]\n" + + "line 2:88: argument of [min(x)] must be [datetime or numeric except unsigned_long or counter_long or counter_integer" + + " or counter_double], found value [min(x)] type [unsigned_long]\n" + + "line 2:96: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], " + + "found value [x] type [unsigned_long]\n" + + "line 2:115: argument of [sum(x)] must be [numeric except unsigned_long or counter_long or counter_integer " + + "or counter_double], found value [x] type [unsigned_long]" ); verifyUnsupported( @@ -1806,16 +1806,16 @@ public void testUnsupportedTypesInStats() { """, "Found 7 problems\n" + "line 2:10: argument of [avg(x)] must be [numeric except unsigned_long], found value [x] type [version]\n" - + "line 2:18: argument of [max(x)] must be [datetime or numeric except unsigned_long or counter_long or " + - "counter_integer or counter_double], found value [max(x)] type [version]\n" + + "line 2:18: argument of [max(x)] must be [datetime or numeric except unsigned_long or counter_long or " + + "counter_integer or counter_double], found value [max(x)] type [version]\n" + "line 2:26: argument of [median(x)] must be [numeric except unsigned_long], found value [x] type [version]\n" - + "line 2:37: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter_long " + - "or counter_integer or counter_double], found value [x] type [version]\n" - + "line 2:67: argument of [min(x)] must be [datetime or numeric except unsigned_long or counter_long or counter_integer " + - "or counter_double], found value [min(x)] type [version]\n" + + "line 2:37: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter_long " + + "or counter_integer or counter_double], found value [x] type [version]\n" + + "line 2:67: argument of [min(x)] must be [datetime or numeric except unsigned_long or counter_long or counter_integer " + + "or counter_double], found value [min(x)] type [version]\n" + "line 2:75: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], found value [x] type [version]\n" - + "line 2:94: argument of [sum(x)] must be [numeric except unsigned_long or counter_long or counter_integer or " + - "counter_double], found value [x] type [version]" + + "line 2:94: argument of [sum(x)] must be [numeric except unsigned_long or counter_long or counter_integer or " + + "counter_double], found value [x] type [version]" ); } From 96bb7b7a3308b8d84454a1c99de70259a71a99c1 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 25 Apr 2024 11:14:32 -0700 Subject: [PATCH 11/13] stylecheck --- .../org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 4af28b6b41f93..eb89c489e41c1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -1813,7 +1813,8 @@ public void testUnsupportedTypesInStats() { + "or counter_integer or counter_double], found value [x] type [version]\n" + "line 2:67: argument of [min(x)] must be [datetime or numeric except unsigned_long or counter_long or counter_integer " + "or counter_double], found value [min(x)] type [version]\n" - + "line 2:75: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], found value [x] type [version]\n" + + "line 2:75: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], " + + "found value [x] type [version]\n" + "line 2:94: argument of [sum(x)] must be [numeric except unsigned_long or counter_long or counter_integer or " + "counter_double], found value [x] type [version]" ); From 165385ef238809644ffcb92228afce0ea1bfdaa4 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Fri, 26 Apr 2024 10:09:04 -0700 Subject: [PATCH 12/13] compact message --- .../expression/function/aggregate/Avg.java | 2 +- .../expression/function/aggregate/Count.java | 8 +-- .../function/aggregate/CountDistinct.java | 2 +- .../expression/function/aggregate/Median.java | 2 +- .../function/aggregate/NumericAggregate.java | 4 +- .../xpack/esql/analysis/AnalyzerTests.java | 60 ++++++++++--------- .../xpack/esql/analysis/VerifierTests.java | 21 ++++--- 7 files changed, 48 insertions(+), 51 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Avg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Avg.java index 3ea0721d52c00..c62551a8aa1f6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Avg.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Avg.java @@ -38,7 +38,7 @@ protected Expression.TypeResolution resolveType() { dt -> dt.isNumeric() && dt != DataTypes.UNSIGNED_LONG, sourceText(), DEFAULT, - "numeric except unsigned_long" + "numeric except unsigned_long or counter types" ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java index 6aa4bd70e42c6..957f83453cac3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java @@ -92,13 +92,7 @@ public Nullability nullable() { @Override protected TypeResolution resolveType() { - return isType( - field(), - dt -> EsqlDataTypes.isCounterType(dt) == false, - sourceText(), - DEFAULT, - "any type except counter_long or counter_integer or counter_double" - ); + return isType(field(), dt -> EsqlDataTypes.isCounterType(dt) == false, sourceText(), DEFAULT, "any type except counter types"); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java index 5e62102aceeaf..b63c070a90ec8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java @@ -90,7 +90,7 @@ protected TypeResolution resolveType() { dt -> resolved && dt != DataTypes.UNSIGNED_LONG, sourceText(), DEFAULT, - "any exact type except unsigned_long" + "any exact type except unsigned_long or counter types" ); if (resolution.unresolved() || precision == null) { return resolution; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java index eb602df21d9a0..8ca3889352e40 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Median.java @@ -44,7 +44,7 @@ protected Expression.TypeResolution resolveType() { dt -> dt.isNumeric() && dt != DataTypes.UNSIGNED_LONG, sourceText(), DEFAULT, - "numeric except unsigned_long" + "numeric except unsigned_long or counter types" ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java index cd86f51bb1fd6..799ec58a18a5d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java @@ -40,7 +40,7 @@ protected TypeResolution resolveType() { sourceText(), DEFAULT, "datetime", - "numeric except unsigned_long or counter_long or counter_integer or counter_double" + "numeric except unsigned_long or counter types" ); } return isType( @@ -48,7 +48,7 @@ protected TypeResolution resolveType() { dt -> dt.isNumeric() && dt != DataTypes.UNSIGNED_LONG, sourceText(), DEFAULT, - "numeric except unsigned_long or counter_long or counter_integer or counter_double" + "numeric except unsigned_long or counter types" ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index eb89c489e41c1..3d8774ba2a649 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -1782,21 +1782,24 @@ public void testUnsupportedTypesInStats() { row x = to_unsigned_long(\"10\") | stats avg(x), count_distinct(x), max(x), median(x), median_absolute_deviation(x), min(x), percentile(x, 10), sum(x) """, - "Found 8 problems\n" - + "line 2:12: argument of [avg(x)] must be [numeric except unsigned_long], found value [x] type [unsigned_long]\n" - + "line 2:20: argument of [count_distinct(x)] must be [any exact type except unsigned_long], found value [x] " - + "type [unsigned_long]\n" - + "line 2:39: argument of [max(x)] must be [datetime or numeric except unsigned_long or counter_long or " - + "counter_integer or counter_double], found value [max(x)] type [unsigned_long]\n" - + "line 2:47: argument of [median(x)] must be [numeric except unsigned_long], found value [x] type [unsigned_long]\n" - + "line 2:58: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter_long or " - + "counter_integer or counter_double], found value [x] type [unsigned_long]\n" - + "line 2:88: argument of [min(x)] must be [datetime or numeric except unsigned_long or counter_long or counter_integer" - + " or counter_double], found value [min(x)] type [unsigned_long]\n" - + "line 2:96: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], " - + "found value [x] type [unsigned_long]\n" - + "line 2:115: argument of [sum(x)] must be [numeric except unsigned_long or counter_long or counter_integer " - + "or counter_double], found value [x] type [unsigned_long]" + """ + Found 8 problems + line 2:12: argument of [avg(x)] must be [numeric except unsigned_long or counter types],\ + found value [x] type [unsigned_long] + line 2:20: argument of [count_distinct(x)] must be [any exact type except unsigned_long or counter types],\ + found value [x] type [unsigned_long] + line 2:39: argument of [max(x)] must be [datetime or numeric except unsigned_long or counter types],\ + found value [max(x)] type [unsigned_long] + line 2:47: argument of [median(x)] must be [numeric except unsigned_long or counter types],\ + found value [x] type [unsigned_long] + line 2:58: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter types],\ + found value [x] type [unsigned_long] + line 2:88: argument of [min(x)] must be [datetime or numeric except unsigned_long or counter types],\ + found value [min(x)] type [unsigned_long] + line 2:96: first argument of [percentile(x, 10)] must be [numeric except unsigned_long],\ + found value [x] type [unsigned_long] + line 2:115: argument of [sum(x)] must be [numeric except unsigned_long or counter types],\ + found value [x] type [unsigned_long]""" ); verifyUnsupported( @@ -1804,19 +1807,20 @@ public void testUnsupportedTypesInStats() { row x = to_version("1.2") | stats avg(x), max(x), median(x), median_absolute_deviation(x), min(x), percentile(x, 10), sum(x) """, - "Found 7 problems\n" - + "line 2:10: argument of [avg(x)] must be [numeric except unsigned_long], found value [x] type [version]\n" - + "line 2:18: argument of [max(x)] must be [datetime or numeric except unsigned_long or counter_long or " - + "counter_integer or counter_double], found value [max(x)] type [version]\n" - + "line 2:26: argument of [median(x)] must be [numeric except unsigned_long], found value [x] type [version]\n" - + "line 2:37: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter_long " - + "or counter_integer or counter_double], found value [x] type [version]\n" - + "line 2:67: argument of [min(x)] must be [datetime or numeric except unsigned_long or counter_long or counter_integer " - + "or counter_double], found value [min(x)] type [version]\n" - + "line 2:75: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], " - + "found value [x] type [version]\n" - + "line 2:94: argument of [sum(x)] must be [numeric except unsigned_long or counter_long or counter_integer or " - + "counter_double], found value [x] type [version]" + """ + Found 7 problems + line 2:10: argument of [avg(x)] must be [numeric except unsigned_long or counter types],\ + found value [x] type [version] + line 2:18: argument of [max(x)] must be [datetime or numeric except unsigned_long or counter types],\ + found value [max(x)] type [version] + line 2:26: argument of [median(x)] must be [numeric except unsigned_long or counter types],\ + found value [x] type [version] + line 2:37: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter types],\ + found value [x] type [version] + line 2:67: argument of [min(x)] must be [datetime or numeric except unsigned_long or counter types],\ + found value [min(x)] type [version] + line 2:75: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], found value [x] type [version] + line 2:94: argument of [sum(x)] must be [numeric except unsigned_long or counter types], found value [x] type [version]""" ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 193c9cea6dc5e..8495e8620dd9e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -74,7 +74,8 @@ public void testAggsExpressionsInStatsAggs() { error("from test | stats max(max(salary)) by first_name") ); assertEquals( - "1:25: argument of [avg(first_name)] must be [numeric except unsigned_long], found value [first_name] type [keyword]", + "1:25: argument of [avg(first_name)] must be [numeric except unsigned_long or counter types]," + + " found value [first_name] type [keyword]", error("from test | stats count(avg(first_name)) by first_name") ); assertEquals( @@ -380,8 +381,8 @@ public void testUnsignedLongNegation() { public void testSumOnDate() { assertEquals( - "1:19: argument of [sum(hire_date)] must be [numeric except unsigned_long or counter_long or" - + " counter_integer or counter_double], found value [hire_date] type [datetime]", + "1:19: argument of [sum(hire_date)] must be [numeric except unsigned_long or counter types]," + + " found value [hire_date] type [datetime]", error("from test | stats sum(hire_date)") ); } @@ -487,26 +488,24 @@ public void testAggregateOnCounter() { assertThat( error("FROM tests | STATS min(network.bytes_in)", tsdb), equalTo( - "1:20: argument of [min(network.bytes_in)] must be " - + "[datetime or numeric except unsigned_long or counter_long or counter_integer or counter_double], " - + "found value [min(network.bytes_in)] type [counter_long]" + "1:20: argument of [min(network.bytes_in)] must be [datetime or numeric except unsigned_long or counter types]," + + " found value [min(network.bytes_in)] type [counter_long]" ) ); assertThat( error("FROM tests | STATS max(network.bytes_in)", tsdb), equalTo( - "1:20: argument of [max(network.bytes_in)] must be " - + "[datetime or numeric except unsigned_long or counter_long or counter_integer or counter_double], " - + "found value [max(network.bytes_in)] type [counter_long]" + "1:20: argument of [max(network.bytes_in)] must be [datetime or numeric except unsigned_long or counter types]," + + " found value [max(network.bytes_in)] type [counter_long]" ) ); assertThat( error("FROM tests | STATS count(network.bytes_out)", tsdb), equalTo( - "1:20: argument of [count(network.bytes_out)] must be [any type except counter_long or counter_integer" - + " or counter_double], found value [network.bytes_out] type [counter_long]" + "1:20: argument of [count(network.bytes_out)] must be [any type except counter types]," + + " found value [network.bytes_out] type [counter_long]" ) ); } From 84733ff2060dec7600d94fde075432d6522d3433 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Fri, 26 Apr 2024 10:42:58 -0700 Subject: [PATCH 13/13] allow widen to_double and to_long --- .../kibana/definition/to_double.json | 24 ++++++ .../functions/kibana/definition/to_long.json | 12 +++ .../esql/functions/types/to_double.asciidoc | 2 + .../esql/functions/types/to_long.asciidoc | 1 + .../src/main/resources/meta.csv-spec | 12 +-- .../function/scalar/convert/ToDouble.java | 17 +++- .../function/scalar/convert/ToLong.java | 15 +++- .../xpack/esql/analysis/AnalyzerTests.java | 82 +++++++++---------- .../xpack/esql/analysis/VerifierTests.java | 4 +- .../function/AbstractFunctionTestCase.java | 11 ++- .../scalar/convert/ToDoubleTests.java | 16 ++++ .../function/scalar/convert/ToLongTests.java | 8 ++ 12 files changed, 145 insertions(+), 59 deletions(-) diff --git a/docs/reference/esql/functions/kibana/definition/to_double.json b/docs/reference/esql/functions/kibana/definition/to_double.json index 362fd1863a68f..f4e414068db61 100644 --- a/docs/reference/esql/functions/kibana/definition/to_double.json +++ b/docs/reference/esql/functions/kibana/definition/to_double.json @@ -28,6 +28,30 @@ "variadic" : false, "returnType" : "double" }, + { + "params" : [ + { + "name" : "field", + "type" : "counter_integer", + "optional" : false, + "description" : "Input value. The input can be a single- or multi-valued column or an expression." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "field", + "type" : "counter_long", + "optional" : false, + "description" : "Input value. The input can be a single- or multi-valued column or an expression." + } + ], + "variadic" : false, + "returnType" : "double" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/kibana/definition/to_long.json b/docs/reference/esql/functions/kibana/definition/to_long.json index 466de6800509d..e3218eba9642a 100644 --- a/docs/reference/esql/functions/kibana/definition/to_long.json +++ b/docs/reference/esql/functions/kibana/definition/to_long.json @@ -16,6 +16,18 @@ "variadic" : false, "returnType" : "long" }, + { + "params" : [ + { + "name" : "field", + "type" : "counter_integer", + "optional" : false, + "description" : "Input value. The input can be a single- or multi-valued column or an expression." + } + ], + "variadic" : false, + "returnType" : "long" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/types/to_double.asciidoc b/docs/reference/esql/functions/types/to_double.asciidoc index a9adcfa2eeb42..cff686c7bc4ca 100644 --- a/docs/reference/esql/functions/types/to_double.asciidoc +++ b/docs/reference/esql/functions/types/to_double.asciidoc @@ -7,6 +7,8 @@ field | result boolean | double counter_double | double +counter_integer | double +counter_long | double datetime | double double | double integer | double diff --git a/docs/reference/esql/functions/types/to_long.asciidoc b/docs/reference/esql/functions/types/to_long.asciidoc index bb198dfa5abb9..b3959c5444e34 100644 --- a/docs/reference/esql/functions/types/to_long.asciidoc +++ b/docs/reference/esql/functions/types/to_long.asciidoc @@ -6,6 +6,7 @@ |=== field | result boolean | long +counter_integer | long counter_long | long datetime | long double | long diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec index 9f133c140853b..f038e9e54c9a6 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec @@ -85,16 +85,16 @@ double tau() "cartesian_point to_cartesianpoint(field:cartesian_point|keyword|text)" "cartesian_shape to_cartesianshape(field:cartesian_point|cartesian_shape|keyword|text)" "date to_datetime(field:date|keyword|text|double|long|unsigned_long|integer)" -"double to_dbl(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double)" +"double to_dbl(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long)" "double to_degrees(number:double|integer|long|unsigned_long)" -"double to_double(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double)" +"double to_double(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long)" "date to_dt(field:date|keyword|text|double|long|unsigned_long|integer)" "geo_point to_geopoint(field:geo_point|keyword|text)" "geo_shape to_geoshape(field:geo_point|geo_shape|keyword|text)" "integer to_int(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer)" "integer to_integer(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer)" "ip to_ip(field:ip|keyword|text)" -"long to_long(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_long)" +"long to_long(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer|counter_long)" "keyword|text to_lower(str:keyword|text)" "double to_radians(number:double|integer|long|unsigned_long)" "keyword to_str(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)" @@ -198,16 +198,16 @@ to_boolean |field |"boolean|keyword|text|double to_cartesianpo|field |"cartesian_point|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. to_cartesiansh|field |"cartesian_point|cartesian_shape|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. to_datetime |field |"date|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. -to_dbl |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double" |Input value. The input can be a single- or multi-valued column or an expression. +to_dbl |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long" |Input value. The input can be a single- or multi-valued column or an expression. to_degrees |number |"double|integer|long|unsigned_long" |Input value. The input can be a single- or multi-valued column or an expression. -to_double |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double" |Input value. The input can be a single- or multi-valued column or an expression. +to_double |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long" |Input value. The input can be a single- or multi-valued column or an expression. to_dt |field |"date|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression. to_geopoint |field |"geo_point|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. to_geoshape |field |"geo_point|geo_shape|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. to_int |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer" |Input value. The input can be a single- or multi-valued column or an expression. to_integer |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer" |Input value. The input can be a single- or multi-valued column or an expression. to_ip |field |"ip|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression. -to_long |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_long" |Input value. The input can be a single- or multi-valued column or an expression. +to_long |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer|counter_long" |Input value. The input can be a single- or multi-valued column or an expression. to_lower |str |"keyword|text" |String expression. If `null`, the function returns `null`. to_radians |number |"double|integer|long|unsigned_long" |Input value. The input can be a single- or multi-valued column or an expression. to_str |field |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version" |Input value. The input can be a single- or multi-valued column or an expression. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java index 8e837dde9e387..20cb46def4d8b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java @@ -44,7 +44,9 @@ public class ToDouble extends AbstractConvertFunction { Map.entry(UNSIGNED_LONG, ToDoubleFromUnsignedLongEvaluator.Factory::new), Map.entry(LONG, ToDoubleFromLongEvaluator.Factory::new), // CastLongToDoubleEvaluator would be a candidate, but not MV'd Map.entry(INTEGER, ToDoubleFromIntEvaluator.Factory::new), // CastIntToDoubleEvaluator would be a candidate, but not MV'd - Map.entry(EsqlDataTypes.COUNTER_DOUBLE, (field, source) -> field) + Map.entry(EsqlDataTypes.COUNTER_DOUBLE, (field, source) -> field), + Map.entry(EsqlDataTypes.COUNTER_INTEGER, ToDoubleFromIntEvaluator.Factory::new), + Map.entry(EsqlDataTypes.COUNTER_LONG, ToDoubleFromLongEvaluator.Factory::new) ); @FunctionInfo( @@ -67,7 +69,18 @@ public ToDouble( Source source, @Param( name = "field", - type = { "boolean", "date", "keyword", "text", "double", "long", "unsigned_long", "integer", "counter_double" }, + type = { + "boolean", + "date", + "keyword", + "text", + "double", + "long", + "unsigned_long", + "integer", + "counter_double", + "counter_integer", + "counter_long" }, description = "Input value. The input can be a single- or multi-valued column or an expression." ) Expression field ) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java index 9b8f01542e80a..c7b77a3c7f2c6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java @@ -45,7 +45,8 @@ public class ToLong extends AbstractConvertFunction { Map.entry(DOUBLE, ToLongFromDoubleEvaluator.Factory::new), Map.entry(UNSIGNED_LONG, ToLongFromUnsignedLongEvaluator.Factory::new), Map.entry(INTEGER, ToLongFromIntEvaluator.Factory::new), // CastIntToLongEvaluator would be a candidate, but not MV'd - Map.entry(EsqlDataTypes.COUNTER_LONG, (field, source) -> field) + Map.entry(EsqlDataTypes.COUNTER_LONG, (field, source) -> field), + Map.entry(EsqlDataTypes.COUNTER_INTEGER, ToLongFromIntEvaluator.Factory::new) ); @FunctionInfo( @@ -69,7 +70,17 @@ public ToLong( Source source, @Param( name = "field", - type = { "boolean", "date", "keyword", "text", "double", "long", "unsigned_long", "integer", "counter_long" }, + type = { + "boolean", + "date", + "keyword", + "text", + "double", + "long", + "unsigned_long", + "integer", + "counter_integer", + "counter_long" }, description = "Input value. The input can be a single- or multi-valued column or an expression." ) Expression field ) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 3d8774ba2a649..1f32a5a76f3e8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -1777,51 +1777,45 @@ public void testDeferredGroupingInStats() { } public void testUnsupportedTypesInStats() { - verifyUnsupported( - """ - row x = to_unsigned_long(\"10\") - | stats avg(x), count_distinct(x), max(x), median(x), median_absolute_deviation(x), min(x), percentile(x, 10), sum(x) - """, - """ - Found 8 problems - line 2:12: argument of [avg(x)] must be [numeric except unsigned_long or counter types],\ - found value [x] type [unsigned_long] - line 2:20: argument of [count_distinct(x)] must be [any exact type except unsigned_long or counter types],\ - found value [x] type [unsigned_long] - line 2:39: argument of [max(x)] must be [datetime or numeric except unsigned_long or counter types],\ - found value [max(x)] type [unsigned_long] - line 2:47: argument of [median(x)] must be [numeric except unsigned_long or counter types],\ - found value [x] type [unsigned_long] - line 2:58: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter types],\ - found value [x] type [unsigned_long] - line 2:88: argument of [min(x)] must be [datetime or numeric except unsigned_long or counter types],\ - found value [min(x)] type [unsigned_long] - line 2:96: first argument of [percentile(x, 10)] must be [numeric except unsigned_long],\ - found value [x] type [unsigned_long] - line 2:115: argument of [sum(x)] must be [numeric except unsigned_long or counter types],\ - found value [x] type [unsigned_long]""" - ); + verifyUnsupported(""" + row x = to_unsigned_long(\"10\") + | stats avg(x), count_distinct(x), max(x), median(x), median_absolute_deviation(x), min(x), percentile(x, 10), sum(x) + """, """ + Found 8 problems + line 2:12: argument of [avg(x)] must be [numeric except unsigned_long or counter types],\ + found value [x] type [unsigned_long] + line 2:20: argument of [count_distinct(x)] must be [any exact type except unsigned_long or counter types],\ + found value [x] type [unsigned_long] + line 2:39: argument of [max(x)] must be [datetime or numeric except unsigned_long or counter types],\ + found value [max(x)] type [unsigned_long] + line 2:47: argument of [median(x)] must be [numeric except unsigned_long or counter types],\ + found value [x] type [unsigned_long] + line 2:58: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter types],\ + found value [x] type [unsigned_long] + line 2:88: argument of [min(x)] must be [datetime or numeric except unsigned_long or counter types],\ + found value [min(x)] type [unsigned_long] + line 2:96: first argument of [percentile(x, 10)] must be [numeric except unsigned_long],\ + found value [x] type [unsigned_long] + line 2:115: argument of [sum(x)] must be [numeric except unsigned_long or counter types],\ + found value [x] type [unsigned_long]"""); - verifyUnsupported( - """ - row x = to_version("1.2") - | stats avg(x), max(x), median(x), median_absolute_deviation(x), min(x), percentile(x, 10), sum(x) - """, - """ - Found 7 problems - line 2:10: argument of [avg(x)] must be [numeric except unsigned_long or counter types],\ - found value [x] type [version] - line 2:18: argument of [max(x)] must be [datetime or numeric except unsigned_long or counter types],\ - found value [max(x)] type [version] - line 2:26: argument of [median(x)] must be [numeric except unsigned_long or counter types],\ - found value [x] type [version] - line 2:37: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter types],\ - found value [x] type [version] - line 2:67: argument of [min(x)] must be [datetime or numeric except unsigned_long or counter types],\ - found value [min(x)] type [version] - line 2:75: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], found value [x] type [version] - line 2:94: argument of [sum(x)] must be [numeric except unsigned_long or counter types], found value [x] type [version]""" - ); + verifyUnsupported(""" + row x = to_version("1.2") + | stats avg(x), max(x), median(x), median_absolute_deviation(x), min(x), percentile(x, 10), sum(x) + """, """ + Found 7 problems + line 2:10: argument of [avg(x)] must be [numeric except unsigned_long or counter types],\ + found value [x] type [version] + line 2:18: argument of [max(x)] must be [datetime or numeric except unsigned_long or counter types],\ + found value [max(x)] type [version] + line 2:26: argument of [median(x)] must be [numeric except unsigned_long or counter types],\ + found value [x] type [version] + line 2:37: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter types],\ + found value [x] type [version] + line 2:67: argument of [min(x)] must be [datetime or numeric except unsigned_long or counter types],\ + found value [min(x)] type [version] + line 2:75: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], found value [x] type [version] + line 2:94: argument of [sum(x)] must be [numeric except unsigned_long or counter types], found value [x] type [version]"""); } public void testInOnText() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 8495e8620dd9e..f563e1a6cb25c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -504,8 +504,8 @@ public void testAggregateOnCounter() { assertThat( error("FROM tests | STATS count(network.bytes_out)", tsdb), equalTo( - "1:20: argument of [count(network.bytes_out)] must be [any type except counter types]," + - " found value [network.bytes_out] type [counter_long]" + "1:20: argument of [count(network.bytes_out)] must be [any type except counter types]," + + " found value [network.bytes_out] type [counter_long]" ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index 32454333d4621..772dea0ef4557 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -946,7 +946,7 @@ protected static String typeErrorMessage(boolean includeOrdinal, List parameters() { l -> l, List.of() ); + TestCaseSupplier.unary( + suppliers, + evaluatorName.apply("Integer"), + List.of(new TestCaseSupplier.TypedDataSupplier("counter", () -> randomInt(1000), EsqlDataTypes.COUNTER_INTEGER)), + DataTypes.DOUBLE, + l -> ((Integer) l).doubleValue(), + List.of() + ); + TestCaseSupplier.unary( + suppliers, + evaluatorName.apply("Long"), + List.of(new TestCaseSupplier.TypedDataSupplier("counter", () -> randomLongBetween(1, 1000), EsqlDataTypes.COUNTER_LONG)), + DataTypes.DOUBLE, + l -> ((Long) l).doubleValue(), + List.of() + ); return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java index cd515d4afd3c9..3b123344b4b11 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java @@ -218,6 +218,14 @@ public static Iterable parameters() { l -> l, List.of() ); + TestCaseSupplier.unary( + suppliers, + evaluatorName.apply("Integer"), + List.of(new TestCaseSupplier.TypedDataSupplier("counter", ESTestCase::randomInt, EsqlDataTypes.COUNTER_INTEGER)), + DataTypes.LONG, + l -> ((Integer) l).longValue(), + List.of() + ); return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); }