Support metrics counter types in ESQL

docs/changelog/107877.yaml

@@ -0,0 +1,5 @@
+pr: 107877
+summary: Support metrics counter types in ESQL
+area: "ES|QL"
+type: enhancement
+issues: []

docs/reference/esql/functions/kibana/definition/to_double.json

@@ -16,6 +16,18 @@
       "variadic" : false,
       "returnType" : "double"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "counter_double",
+          "optional" : false,
+          "description" : "Input value. The input can be a single- or multi-valued column or an expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "double"
+    },
     {
       "params" : [
         {

docs/reference/esql/functions/kibana/definition/to_integer.json

@@ -16,6 +16,18 @@
       "variadic" : false,
       "returnType" : "integer"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "counter_integer",
+          "optional" : false,
+          "description" : "Input value. The input can be a single- or multi-valued column or an expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "integer"
+    },
     {
       "params" : [
         {

docs/reference/esql/functions/kibana/definition/to_long.json

@@ -16,6 +16,18 @@
       "variadic" : false,
       "returnType" : "long"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "counter_long",
+          "optional" : false,
+          "description" : "Input value. The input can be a single- or multi-valued column or an expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
     {
       "params" : [
         {

docs/reference/esql/functions/types/to_double.asciidoc

@@ -6,6 +6,7 @@
 |===
 field | result
 boolean | double
+counter_double | double
 datetime | double
 double | double
 integer | double

docs/reference/esql/functions/types/to_integer.asciidoc

@@ -6,6 +6,7 @@
 |===
 field | result
 boolean | integer
+counter_integer | integer
 datetime | integer
 double | integer
 integer | integer

docs/reference/esql/functions/types/to_long.asciidoc

@@ -6,6 +6,7 @@
 |===
 field | result
 boolean | long
+counter_long | long
 datetime | long
 double | long
 integer | long

server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

@@ -1709,10 +1709,6 @@ public Function<byte[], Number> pointReaderIfPossible() {
 
         @Override
         public BlockLoader blockLoader(BlockLoaderContext blContext) {
-            if (indexMode == IndexMode.TIME_SERIES && metricType == TimeSeriesParams.MetricType.COUNTER) {
-                // Counters are not supported by ESQL so we load them in null
-                return BlockLoader.CONSTANT_NULLS;
-            }
             if (hasDocValues()) {
                 return type.blockLoaderFromDocValues(name());
             }

x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec

@@ -85,16 +85,16 @@ double tau()
 "cartesian_point to_cartesianpoint(field:cartesian_point|keyword|text)"
 "cartesian_shape to_cartesianshape(field:cartesian_point|cartesian_shape|keyword|text)"
 "date to_datetime(field:date|keyword|text|double|long|unsigned_long|integer)"
-"double to_dbl(field:boolean|date|keyword|text|double|long|unsigned_long|integer)"
+"double to_dbl(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double)"
 "double to_degrees(number:double|integer|long|unsigned_long)"
-"double to_double(field:boolean|date|keyword|text|double|long|unsigned_long|integer)"
+"double to_double(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double)"
 "date to_dt(field:date|keyword|text|double|long|unsigned_long|integer)"
 "geo_point to_geopoint(field:geo_point|keyword|text)"
 "geo_shape to_geoshape(field:geo_point|geo_shape|keyword|text)"
-"integer to_int(field:boolean|date|keyword|text|double|long|unsigned_long|integer)"
-"integer to_integer(field:boolean|date|keyword|text|double|long|unsigned_long|integer)"
+"integer to_int(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer)"
+"integer to_integer(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer)"
 "ip to_ip(field:ip|keyword|text)"
-"long to_long(field:boolean|date|keyword|text|double|long|unsigned_long|integer)"
+"long to_long(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_long)"
 "keyword|text to_lower(str:keyword|text)"
 "double to_radians(number:double|integer|long|unsigned_long)"
 "keyword to_str(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
@@ -198,16 +198,16 @@ to_boolean    |field                               |"boolean|keyword|text|double
 to_cartesianpo|field                               |"cartesian_point|keyword|text"                                                                                                    |Input value. The input can be a single- or multi-valued column or an expression.
 to_cartesiansh|field                               |"cartesian_point|cartesian_shape|keyword|text"                                                                                    |Input value. The input can be a single- or multi-valued column or an expression.
 to_datetime   |field                               |"date|keyword|text|double|long|unsigned_long|integer"                                                                             |Input value. The input can be a single- or multi-valued column or an expression.
-to_dbl        |field                               |"boolean|date|keyword|text|double|long|unsigned_long|integer"                                                                     |Input value. The input can be a single- or multi-valued column or an expression.
+to_dbl        |field                               |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double"                                                      |Input value. The input can be a single- or multi-valued column or an expression.
 to_degrees    |number                              |"double|integer|long|unsigned_long"                                                                                               |Input value. The input can be a single- or multi-valued column or an expression.
-to_double     |field                               |"boolean|date|keyword|text|double|long|unsigned_long|integer"                                                                     |Input value. The input can be a single- or multi-valued column or an expression.
+to_double     |field                               |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double"                                                      |Input value. The input can be a single- or multi-valued column or an expression.
 to_dt         |field                               |"date|keyword|text|double|long|unsigned_long|integer"                                                                             |Input value. The input can be a single- or multi-valued column or an expression.
 to_geopoint   |field                               |"geo_point|keyword|text"                                                                                                          |Input value. The input can be a single- or multi-valued column or an expression.
 to_geoshape   |field                               |"geo_point|geo_shape|keyword|text"                                                                                                |Input value. The input can be a single- or multi-valued column or an expression.
-to_int        |field                               |"boolean|date|keyword|text|double|long|unsigned_long|integer"                                                                     |Input value. The input can be a single- or multi-valued column or an expression.
-to_integer    |field                               |"boolean|date|keyword|text|double|long|unsigned_long|integer"                                                                     |Input value. The input can be a single- or multi-valued column or an expression.
+to_int        |field                               |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer"                                                     |Input value. The input can be a single- or multi-valued column or an expression.
+to_integer    |field                               |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer"                                                     |Input value. The input can be a single- or multi-valued column or an expression.
 to_ip         |field                               |"ip|keyword|text"                                                                                                                 |Input value. The input can be a single- or multi-valued column or an expression.
-to_long       |field                               |"boolean|date|keyword|text|double|long|unsigned_long|integer"                                                                     |Input value. The input can be a single- or multi-valued column or an expression.
+to_long       |field                               |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_long"                                                        |Input value. The input can be a single- or multi-valued column or an expression.
 to_lower      |str                                 |"keyword|text"                                                                                                                    |String expression. If `null`, the function returns `null`.
 to_radians    |number                              |"double|integer|long|unsigned_long"                                                                                               |Input value. The input can be a single- or multi-valued column or an expression.
 to_str        |field                               |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version"      |Input value. The input can be a single- or multi-valued column or an expression.

x-pack/plugin/esql/qa/testFixtures/src/main/resources/tsdb-mapping.json

@@ -0,0 +1,30 @@
+{
+  "properties": {
+    "@timestamp": {
+      "type": "date"
+    },
+    "metricset": {
+      "type": "keyword",
+      "time_series_dimension": true
+    },
+    "name": {
+      "type": "keyword"
+    },
+    "network": {
+      "properties": {
+        "connections": {
+          "type": "long",
+          "time_series_metric": "gauge"
+        },
+        "bytes_in": {
+          "type": "long",
+          "time_series_metric": "counter"
+        },
+        "bytes_out": {
+          "type": "long",
+          "time_series_metric": "counter"
+        }
+      }
+    }
+  }
+}

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java

@@ -61,21 +61,21 @@ protected abstract XContentBuilder valueToXContent(XContentBuilder builder, ToXC
 
     public static PositionToXContent positionToXContent(ColumnInfo columnInfo, Block block, BytesRef scratch) {
         return switch (columnInfo.type()) {
-            case "long" -> new PositionToXContent(block) {
+            case "long", "counter_long" -> new PositionToXContent(block) {
                 @Override
                 protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Params params, int valueIndex)
                     throws IOException {
                     return builder.value(((LongBlock) block).getLong(valueIndex));
                 }
             };
-            case "integer" -> new PositionToXContent(block) {
+            case "integer", "counter_integer" -> new PositionToXContent(block) {
                 @Override
                 protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Params params, int valueIndex)
                     throws IOException {
                     return builder.value(((IntBlock) block).getInt(valueIndex));
                 }
             };
-            case "double" -> new PositionToXContent(block) {
+            case "double", "counter_double" -> new PositionToXContent(block) {
                 @Override
                 protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Params params, int valueIndex)
                     throws IOException {

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java

@@ -123,9 +123,9 @@ static Object valueAtPosition(Block block, int position, String dataType, BytesR
     private static Object valueAt(String dataType, Block block, int offset, BytesRef scratch) {
         return switch (dataType) {
             case "unsigned_long" -> unsignedLongAsNumber(((LongBlock) block).getLong(offset));
-            case "long" -> ((LongBlock) block).getLong(offset);
-            case "integer" -> ((IntBlock) block).getInt(offset);
-            case "double" -> ((DoubleBlock) block).getDouble(offset);
+            case "long", "counter_long" -> ((LongBlock) block).getLong(offset);
+            case "integer", "counter_integer" -> ((IntBlock) block).getInt(offset);
+            case "double", "counter_double" -> ((DoubleBlock) block).getDouble(offset);
             case "keyword", "text" -> ((BytesRefBlock) block).getBytesRef(offset, scratch).utf8ToString();
             case "ip" -> {
                 BytesRef val = ((BytesRefBlock) block).getBytesRef(offset, scratch);
@@ -174,9 +174,9 @@ static Page valuesToPage(BlockFactory blockFactory, List<ColumnInfo> columns, Li
                     case "unsigned_long" -> ((LongBlock.Builder) builder).appendLong(
                         longToUnsignedLong(((Number) value).longValue(), true)
                     );
-                    case "long" -> ((LongBlock.Builder) builder).appendLong(((Number) value).longValue());
-                    case "integer" -> ((IntBlock.Builder) builder).appendInt(((Number) value).intValue());
-                    case "double" -> ((DoubleBlock.Builder) builder).appendDouble(((Number) value).doubleValue());
+                    case "long", "counter_long" -> ((LongBlock.Builder) builder).appendLong(((Number) value).longValue());
+                    case "integer", "counter_integer" -> ((IntBlock.Builder) builder).appendInt(((Number) value).intValue());
+                    case "double", "counter_double" -> ((DoubleBlock.Builder) builder).appendDouble(((Number) value).doubleValue());
                     case "keyword", "text", "unsupported" -> ((BytesRefBlock.Builder) builder).appendBytesRef(
                         new BytesRef(value.toString())
                     );

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java

@@ -28,6 +28,7 @@
 import org.elasticsearch.xpack.ql.expression.AttributeSet;
 import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.expression.Expressions;
+import org.elasticsearch.xpack.ql.expression.FieldAttribute;
 import org.elasticsearch.xpack.ql.expression.NamedExpression;
 import org.elasticsearch.xpack.ql.expression.TypeResolutions;
 import org.elasticsearch.xpack.ql.expression.function.aggregate.AggregateFunction;
@@ -193,6 +194,9 @@ private static void checkAggregate(LogicalPlan p, Set<Failure> failures) {
                 if (attr != null) {
                     groupRefs.add(attr);
                 }
+                if (e instanceof FieldAttribute f && EsqlDataTypes.isCounterType(f.dataType())) {
+                    failures.add(fail(e, "cannot group by on [{}] type for grouping [{}]", f.dataType().typeName(), e.sourceText()));
+                }
             });
 
             // check aggregates - accept only aggregate functions or expressions over grouping

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java

@@ -9,14 +9,14 @@
 
 import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier;
 import org.elasticsearch.compute.aggregation.CountAggregatorFunction;
-import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions;
 import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvCount;
 import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mul;
 import org.elasticsearch.xpack.esql.planner.ToAggregator;
+import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.expression.Literal;
 import org.elasticsearch.xpack.ql.expression.Nullability;
@@ -31,6 +31,7 @@
 import java.util.List;
 
 import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT;
+import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isType;
 
 public class Count extends AggregateFunction implements EnclosedAgg, ToAggregator, SurrogateExpression {
 
@@ -91,7 +92,13 @@ public Nullability nullable() {
 
     @Override
     protected TypeResolution resolveType() {
-        return EsqlTypeResolutions.isExact(field(), sourceText(), DEFAULT);
+        return isType(
+            field(),
+            dt -> EsqlDataTypes.isCounterType(dt) == false,
+            sourceText(),
+            DEFAULT,
+            "any type except counter_long or counter_integer or counter_double"
+        );
     }
 
     @Override

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java

@@ -40,15 +40,15 @@ protected TypeResolution resolveType() {
                 sourceText(),
                 DEFAULT,
                 "datetime",
-                "numeric except unsigned_long"
+                "numeric except unsigned_long or counter_long or counter_integer or counter_double"
             );
         }
         return isType(
             field(),
             dt -> dt.isNumeric() && dt != DataTypes.UNSIGNED_LONG,
             sourceText(),
             DEFAULT,
-            "numeric except unsigned_long"
+            "numeric except unsigned_long or counter_long or counter_integer or counter_double"
         );
     }
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDouble.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.xpack.esql.expression.function.Example;
 import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
 import org.elasticsearch.xpack.esql.expression.function.Param;
+import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
 import org.elasticsearch.xpack.ql.InvalidArgumentException;
 import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.tree.NodeInfo;
@@ -42,7 +43,8 @@ public class ToDouble extends AbstractConvertFunction {
         Map.entry(TEXT, ToDoubleFromStringEvaluator.Factory::new),
         Map.entry(UNSIGNED_LONG, ToDoubleFromUnsignedLongEvaluator.Factory::new),
         Map.entry(LONG, ToDoubleFromLongEvaluator.Factory::new), // CastLongToDoubleEvaluator would be a candidate, but not MV'd
-        Map.entry(INTEGER, ToDoubleFromIntEvaluator.Factory::new) // CastIntToDoubleEvaluator would be a candidate, but not MV'd
+        Map.entry(INTEGER, ToDoubleFromIntEvaluator.Factory::new), // CastIntToDoubleEvaluator would be a candidate, but not MV'd
+        Map.entry(EsqlDataTypes.COUNTER_DOUBLE, (field, source) -> field)
     );
 
     @FunctionInfo(
@@ -65,7 +67,7 @@ public ToDouble(
         Source source,
         @Param(
             name = "field",
-            type = { "boolean", "date", "keyword", "text", "double", "long", "unsigned_long", "integer" },
+            type = { "boolean", "date", "keyword", "text", "double", "long", "unsigned_long", "integer", "counter_double" },
             description = "Input value. The input can be a single- or multi-valued column or an expression."
         ) Expression field
     ) {