From 8e334769956e1eb24ddc5319d0a47f42c0b9eeca Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Fri, 14 Nov 2025 11:27:29 +0100 Subject: [PATCH 01/11] Implement min agg --- .../expression/function/aggregate/Min.java | 16 +++++++++++++-- .../histogram/ExtractHistogramComponent.java | 6 ++++++ .../function/aggregate/MinErrorTests.java | 4 +++- .../function/aggregate/MinTests.java | 20 ++++++++++++++++--- 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java index c50b437867f2f..e698d0e90cee5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java @@ -17,6 +17,7 @@ import org.elasticsearch.compute.aggregation.MinIpAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MinLongAggregatorFunctionSupplier; import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder; +import org.elasticsearch.compute.data.ExponentialHistogramBlock; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -30,6 +31,7 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionType; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateMetricDouble; +import org.elasticsearch.xpack.esql.expression.function.scalar.histogram.ExtractHistogramComponent; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin; import org.elasticsearch.xpack.esql.planner.ToAggregator; @@ -88,7 +90,8 @@ public Min( "keyword", "text", "unsigned_long", - "version" } + "version", + "exponential_histogram" } ) Expression field ) { this(source, field, Literal.TRUE, NO_WINDOW); @@ -135,13 +138,14 @@ protected TypeResolution resolveType() { "string", "version", "aggregate_metric_double", + "exponential_histogram", "numeric except counter types" ); } @Override public DataType dataType() { - if (field().dataType() == DataType.AGGREGATE_METRIC_DOUBLE) { + if (field().dataType() == DataType.AGGREGATE_METRIC_DOUBLE || field().dataType() == DataType.EXPONENTIAL_HISTOGRAM) { return DataType.DOUBLE; } return field().dataType().noText(); @@ -167,6 +171,14 @@ public Expression surrogate() { window() ); } + if (field().dataType() == DataType.EXPONENTIAL_HISTOGRAM) { + return new Min( + source(), + ExtractHistogramComponent.create(source(), field(), ExponentialHistogramBlock.Component.MIN), + filter(), + window() + ); + } return field().foldable() ? new MvMin(source(), field()) : null; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/ExtractHistogramComponent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/ExtractHistogramComponent.java index e4096bebc56e7..1f1bf6fdc62e7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/ExtractHistogramComponent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/ExtractHistogramComponent.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -37,6 +38,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isFoldable; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; /** @@ -78,6 +80,10 @@ private ExtractHistogramComponent(StreamInput in) throws IOException { this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class)); } + public static Expression create(Source source, Expression field, ExponentialHistogramBlock.Component component) { + return new ExtractHistogramComponent(source, field, new Literal(source, component.ordinal(), INTEGER)); + } + Expression field() { return field; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinErrorTests.java index b83f8f89a8542..2c52fcc68fe32 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinErrorTests.java @@ -37,7 +37,9 @@ protected Matcher expectedTypeErrorMatcher(List> validPerP false, validPerPosition, signature, - (v, p) -> "boolean, date, ip, string, version, aggregate_metric_double or numeric except counter types" + ( + v, + p) -> "boolean, date, ip, string, version, aggregate_metric_double, exponential_histogram or numeric except counter types" ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java index 3622bc714d831..d7561e3b66a70 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java @@ -14,6 +14,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder; +import org.elasticsearch.exponentialhistogram.ExponentialHistogram; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -54,7 +55,8 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.ipCases(1, 1000), MultiRowTestCaseSupplier.versionCases(1, 1000), MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.KEYWORD), - MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT) + MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT), + MultiRowTestCaseSupplier.exponentialHistogramCases(1, 100) ).flatMap(List::stream).map(MinTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); FunctionAppliesTo unsignedLongAppliesTo = appliesTo(FunctionAppliesToLifecycle.GA, "9.2.0", "", true); @@ -212,7 +214,8 @@ protected Expression build(Source source, List args) { private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier) { return new TestCaseSupplier(fieldSupplier.name(), List.of(fieldSupplier.type()), () -> { var fieldTypedData = fieldSupplier.get(); - Comparable> expected; + Comparable expected; + DataType expectedReturnType; if (fieldSupplier.type() == DataType.AGGREGATE_METRIC_DOUBLE) { expected = fieldTypedData.multiRowData() @@ -223,18 +226,29 @@ private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier ) .min(Comparator.naturalOrder()) .orElse(null); + expectedReturnType = DataType.DOUBLE; + } else if (fieldSupplier.type() == DataType.EXPONENTIAL_HISTOGRAM) { + expected = fieldTypedData.multiRowData() + .stream() + .map(obj -> (ExponentialHistogram) obj) + .filter(histo -> histo.valueCount() > 0) // only non-empty histogramms have an influence + .map(ExponentialHistogram::min) + .min(Comparator.naturalOrder()) + .orElse(null); + expectedReturnType = DataType.DOUBLE; } else { expected = fieldTypedData.multiRowData() .stream() .map(v -> (Comparable>) v) .min(Comparator.naturalOrder()) .orElse(null); + expectedReturnType = fieldSupplier.type(); } return new TestCaseSupplier.TestCase( List.of(fieldTypedData), standardAggregatorName("Min", fieldSupplier.type()), - fieldSupplier.type() == DataType.AGGREGATE_METRIC_DOUBLE ? DataType.DOUBLE : fieldSupplier.type(), + expectedReturnType, equalTo(expected) ); }); From 2179308245719790872afd3ef03a270331c69497 Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Fri, 14 Nov 2025 11:37:52 +0100 Subject: [PATCH 02/11] Implement max agg --- .../expression/function/aggregate/Max.java | 31 ++++++++++++++++--- .../function/aggregate/MaxErrorTests.java | 4 ++- .../function/aggregate/MaxTests.java | 20 ++++++++++-- .../function/aggregate/MinTests.java | 2 +- 4 files changed, 48 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java index 46a76eb241d5c..2d6d27a54c97e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java @@ -17,6 +17,7 @@ import org.elasticsearch.compute.aggregation.MaxIpAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MaxLongAggregatorFunctionSupplier; import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder; +import org.elasticsearch.compute.data.ExponentialHistogramBlock; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -30,6 +31,7 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionType; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateMetricDouble; +import org.elasticsearch.xpack.esql.expression.function.scalar.histogram.ExtractHistogramComponent; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMax; import org.elasticsearch.xpack.esql.planner.ToAggregator; @@ -59,7 +61,18 @@ public class Max extends AggregateFunction implements ToAggregator, SurrogateExp ); @FunctionInfo( - returnType = { "boolean", "double", "integer", "long", "date", "date_nanos", "ip", "keyword", "unsigned_long", "version" }, + returnType = { + "boolean", + "double", + "integer", + "long", + "date", + "date_nanos", + "ip", + "keyword", + "unsigned_long", + "version", + "exponential_histogram" }, description = "The maximum value of a field.", type = FunctionType.AGGREGATE, examples = { @@ -88,7 +101,8 @@ public Max( "keyword", "text", "unsigned_long", - "version" } + "version", + "exponential_histogram" } ) Expression field ) { this(source, field, Literal.TRUE, NO_WINDOW); @@ -126,7 +140,7 @@ public Max replaceChildren(List newChildren) { protected TypeResolution resolveType() { return TypeResolutions.isType( field(), - dt -> SUPPLIERS.containsKey(dt) || dt == DataType.AGGREGATE_METRIC_DOUBLE, + dt -> SUPPLIERS.containsKey(dt) || dt == DataType.AGGREGATE_METRIC_DOUBLE || dt == DataType.EXPONENTIAL_HISTOGRAM, sourceText(), DEFAULT, "boolean", @@ -135,13 +149,14 @@ protected TypeResolution resolveType() { "string", "version", "aggregate_metric_double", + "exponential_histogram", "numeric except counter types" ); } @Override public DataType dataType() { - if (field().dataType() == DataType.AGGREGATE_METRIC_DOUBLE) { + if (field().dataType() == DataType.AGGREGATE_METRIC_DOUBLE || field().dataType() == DataType.EXPONENTIAL_HISTOGRAM) { return DataType.DOUBLE; } return field().dataType().noText(); @@ -167,6 +182,14 @@ public Expression surrogate() { window() ); } + if (field().dataType() == DataType.EXPONENTIAL_HISTOGRAM) { + return new Max( + source(), + ExtractHistogramComponent.create(source(), field(), ExponentialHistogramBlock.Component.MAX), + filter(), + window() + ); + } return field().foldable() ? new MvMax(source(), field()) : null; } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxErrorTests.java index 9bbea7ef6d74d..803602647d7c1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxErrorTests.java @@ -37,7 +37,9 @@ protected Matcher expectedTypeErrorMatcher(List> validPerP false, validPerPosition, signature, - (v, p) -> "boolean, date, ip, string, version, aggregate_metric_double or numeric except counter types" + ( + v, + p) -> "boolean, date, ip, string, version, aggregate_metric_double, exponential_histogram or numeric except counter types" ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java index 67748b2faa7d3..4180edf2d3d60 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java @@ -14,6 +14,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder; +import org.elasticsearch.exponentialhistogram.ExponentialHistogram; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -54,7 +55,8 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.ipCases(1, 1000), MultiRowTestCaseSupplier.versionCases(1, 1000), MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.KEYWORD), - MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT) + MultiRowTestCaseSupplier.stringCases(1, 1000, DataType.TEXT), + MultiRowTestCaseSupplier.exponentialHistogramCases(1, 100) ).flatMap(List::stream).map(MaxTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); FunctionAppliesTo unsignedLongAppliesTo = appliesTo(FunctionAppliesToLifecycle.GA, "9.2.0", "", true); @@ -212,7 +214,8 @@ protected Expression build(Source source, List args) { private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier) { return new TestCaseSupplier(fieldSupplier.name(), List.of(fieldSupplier.type()), () -> { var fieldTypedData = fieldSupplier.get(); - Comparable> expected; + Comparable expected; + DataType expectedReturnType; if (fieldSupplier.type() == DataType.AGGREGATE_METRIC_DOUBLE) { expected = fieldTypedData.multiRowData() @@ -223,18 +226,29 @@ private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier ) .max(Comparator.naturalOrder()) .orElse(null); + expectedReturnType = DataType.DOUBLE; + } else if (fieldSupplier.type() == DataType.EXPONENTIAL_HISTOGRAM) { + expected = fieldTypedData.multiRowData() + .stream() + .map(obj -> (ExponentialHistogram) obj) + .filter(histo -> histo.valueCount() > 0) // only non-empty histograms have an influence + .map(ExponentialHistogram::max) + .min(Comparator.naturalOrder()) + .orElse(null); + expectedReturnType = DataType.DOUBLE; } else { expected = fieldTypedData.multiRowData() .stream() .map(v -> (Comparable>) v) .max(Comparator.naturalOrder()) .orElse(null); + expectedReturnType = fieldSupplier.type(); } return new TestCaseSupplier.TestCase( List.of(fieldTypedData), standardAggregatorName("Max", fieldSupplier.type()), - fieldSupplier.type() == DataType.AGGREGATE_METRIC_DOUBLE ? DataType.DOUBLE : fieldSupplier.type(), + expectedReturnType, equalTo(expected) ); }); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java index d7561e3b66a70..37d60b96f98c4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java @@ -231,7 +231,7 @@ private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier expected = fieldTypedData.multiRowData() .stream() .map(obj -> (ExponentialHistogram) obj) - .filter(histo -> histo.valueCount() > 0) // only non-empty histogramms have an influence + .filter(histo -> histo.valueCount() > 0) // only non-empty histograms have an influence .map(ExponentialHistogram::min) .min(Comparator.naturalOrder()) .orElse(null); From 22c3bf848d224644970d1b4cebf0279f2295b3d7 Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Fri, 14 Nov 2025 11:47:54 +0100 Subject: [PATCH 03/11] Add some more percentile CSV tests --- .../resources/exponential_histogram.csv-spec | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/exponential_histogram.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/exponential_histogram.csv-spec index 53e0158f945b4..b35a00da01d1f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/exponential_histogram.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/exponential_histogram.csv-spec @@ -42,3 +42,65 @@ instance-0 | 2.4E-4 | 0.0211404 | 1.0432946 | 6.786232 instance-1 | 2.17E-4 | 6.469E-4 | 0.1422151 | 3.190723 instance-2 | 2.2E-4 | 6.469E-4 | 0.0857672 | 2.7059714542564097 ; + +percentileOnEmptyHistogram +required_capability: exponential_histogram_percentiles_support + +FROM exp_histo_sample | WHERE instance == "dummy-empty" + | STATS p50 = PERCENTILE(responseTime,50) + | KEEP p50 +; + +p50:double +NULL +; + +inlineGroupedPercentiles +required_capability: exponential_histogram_percentiles_support + +FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy") + | INLINE STATS p0 = PERCENTILE(responseTime,0), p50 = PERCENTILE(responseTime,50), p99 = PERCENTILE(responseTime, 99), p100 = PERCENTILE(responseTime,100) BY instance + | EVAL p50 = ROUND(p50, 7), p99 = ROUND(p99, 7) // rounding to avoid floating point precision issues, min and max are exact so no rounding needed + | KEEP instance, p0, p50, p99, p100 + | SORT instance + | LIMIT 5 +; + +instance:keyword | p0:double | p50:double | p99:double | p100:double +instance-0 | 2.4E-4 | 0.0211404 | 1.0432946 | 6.786232 +instance-0 | 2.4E-4 | 0.0211404 | 1.0432946 | 6.786232 +instance-0 | 2.4E-4 | 0.0211404 | 1.0432946 | 6.786232 +instance-0 | 2.4E-4 | 0.0211404 | 1.0432946 | 6.786232 +instance-0 | 2.4E-4 | 0.0211404 | 1.0432946 | 6.786232 +; + +inlineUngroupedPercentiles +required_capability: exponential_histogram_percentiles_support + +FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy") + | INLINE STATS p0 = PERCENTILE(responseTime,0), p50 = PERCENTILE(responseTime,50), p99 = PERCENTILE(responseTime, 99), p100 = PERCENTILE(responseTime,100) + | EVAL p50 = ROUND(p50, 7), p99 = ROUND(p99, 7) // rounding to avoid floating point precision issues, min and max are exact so no rounding needed + | KEEP instance, p0, p50, p99, p100 + | SORT instance + | LIMIT 5 +; + +instance:keyword | p0:double | p50:double | p99:double | p100:double +instance-0 | 2.17E-4 | 0.0016965 | 0.9472324 | 6.786232 +instance-0 | 2.17E-4 | 0.0016965 | 0.9472324 | 6.786232 +instance-0 | 2.17E-4 | 0.0016965 | 0.9472324 | 6.786232 +instance-0 | 2.17E-4 | 0.0016965 | 0.9472324 | 6.786232 +instance-0 | 2.17E-4 | 0.0016965 | 0.9472324 | 6.786232 +; + +inlinePercentileOnEmptyHistogram +required_capability: exponential_histogram_percentiles_support + +FROM exp_histo_sample | WHERE instance == "dummy-empty" + | INLINE STATS p50 = PERCENTILE(responseTime,50) + | KEEP instance, p50 +; + +instance:keyword | p50:double +"dummy-empty" | NULL +; From 3ea5a87890727194e36d77d138de776636f0cc90 Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Fri, 14 Nov 2025 11:52:14 +0100 Subject: [PATCH 04/11] Fix min type verification --- .../xpack/esql/expression/function/aggregate/Min.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java index e698d0e90cee5..83a1d585c8e78 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java @@ -129,7 +129,7 @@ public Min withFilter(Expression filter) { protected TypeResolution resolveType() { return TypeResolutions.isType( field(), - dt -> SUPPLIERS.containsKey(dt) || dt == DataType.AGGREGATE_METRIC_DOUBLE, + dt -> SUPPLIERS.containsKey(dt) || dt == DataType.AGGREGATE_METRIC_DOUBLE || dt == DataType.EXPONENTIAL_HISTOGRAM, sourceText(), DEFAULT, "boolean", From dcaf4d66e276e69d52b7170e3196be2a5597e309 Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Fri, 14 Nov 2025 12:22:29 +0100 Subject: [PATCH 05/11] CSV tests --- .../resources/exponential_histogram.csv-spec | 126 +++++++++++++----- .../xpack/esql/action/EsqlCapabilities.java | 15 ++- 2 files changed, 105 insertions(+), 36 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/exponential_histogram.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/exponential_histogram.csv-spec index b35a00da01d1f..a77fb05b14eb4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/exponential_histogram.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/exponential_histogram.csv-spec @@ -14,6 +14,71 @@ dummy-zero_count_only | "{""scale"":2,""sum"":0.0,""min"":0.0,""max"":0.0,"" dummy-zero_threshold_only | "{""scale"":0,""sum"":0.0,""zero"":{""threshold"":2.0E-5}}" ; + + +allAggsGrouped +required_capability: exponential_histogram_minmax_support + +FROM exp_histo_sample + | EVAL instance = CASE(STARTS_WITH(instance, "dummy"), "dummy-grouped", instance) + | STATS min = MIN(responseTime), max = MAX(responseTime), p75 = PERCENTILE(responseTime,75) BY instance + | EVAL p75 = ROUND(p75, 7) // rounding to avoid floating point precision issues + | KEEP instance, min, max, p75 +; + +instance:keyword | min:double | max:double | p75:double +dummy-grouped | -100.0 | 50.0 | 8.3457089 +instance-2 | 2.2E-4 | 2.744054 | 0.0016068 +instance-0 | 2.4E-4 | 6.786232 | 0.2608237 +instance-1 | 2.17E-4 | 3.190723 | 0.0016068 +; + + + +allAggsInlineGrouped +required_capability: exponential_histogram_minmax_support + +FROM exp_histo_sample + | INLINE STATS min = MIN(responseTime), max = MAX(responseTime), p75 = PERCENTILE(responseTime,75) BY instance + | EVAL p75 = ROUND(p75, 7) // rounding to avoid floating point precision issues + | KEEP instance, min, max, p75 + | Limit 15 +; + +instance:keyword | min:double | max:double | p75:double +dummy-empty | null | null | null +dummy-full | -100.0 | 50.0 | 10.6666667 +dummy-no_zero_bucket | -100.0 | 50.0 | 10.6666667 +dummy-positive_only | 1.0 | 50.0 | 34.7656715 +dummy-negative_only | -50.0 | -1.0 | -12.8729318 +dummy-zero_threshold_only | null | null | null +dummy-zero_count_only | 0.0 | 0.0 | 0.0 +instance-2 | 2.2E-4 | 2.744054 | 0.0016068 +instance-0 | 2.4E-4 | 6.786232 | 0.2608237 +instance-1 | 2.17E-4 | 3.190723 | 0.0016068 +instance-2 | 2.2E-4 | 2.744054 | 0.0016068 +instance-0 | 2.4E-4 | 6.786232 | 0.2608237 +instance-1 | 2.17E-4 | 3.190723 | 0.0016068 +instance-2 | 2.2E-4 | 2.744054 | 0.0016068 +instance-0 | 2.4E-4 | 6.786232 | 0.2608237 +; + + + +allAggsOnEmptyHistogram +required_capability: exponential_histogram_minmax_support + +FROM exp_histo_sample | WHERE instance == "dummy-empty" + | STATS min = MIN(responseTime), max = MAX(responseTime), p75 = PERCENTILE(responseTime,75) + | KEEP min, max, p75 +; + +min:double | max:double | p75:double +NULL | NULL | NULL +; + + + ungroupedPercentiles required_capability: exponential_histogram_percentiles_support @@ -27,6 +92,8 @@ p0:double | p50:double | p99:double | p100:double 2.17E-4 | 0.0016965 | 0.9472324 | 6.786232 ; + + groupedPercentiles required_capability: exponential_histogram_percentiles_support @@ -43,6 +110,8 @@ instance-1 | 2.17E-4 | 6.469E-4 | 0.1422151 | 3.190723 instance-2 | 2.2E-4 | 6.469E-4 | 0.0857672 | 2.7059714542564097 ; + + percentileOnEmptyHistogram required_capability: exponential_histogram_percentiles_support @@ -55,52 +124,47 @@ p50:double NULL ; -inlineGroupedPercentiles -required_capability: exponential_histogram_percentiles_support + + +ungroupedMinMax +required_capability: exponential_histogram_minmax_support FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy") - | INLINE STATS p0 = PERCENTILE(responseTime,0), p50 = PERCENTILE(responseTime,50), p99 = PERCENTILE(responseTime, 99), p100 = PERCENTILE(responseTime,100) BY instance - | EVAL p50 = ROUND(p50, 7), p99 = ROUND(p99, 7) // rounding to avoid floating point precision issues, min and max are exact so no rounding needed - | KEEP instance, p0, p50, p99, p100 - | SORT instance - | LIMIT 5 + | STATS min = MIN(responseTime), max = MAX(responseTime) + | KEEP min, max ; -instance:keyword | p0:double | p50:double | p99:double | p100:double -instance-0 | 2.4E-4 | 0.0211404 | 1.0432946 | 6.786232 -instance-0 | 2.4E-4 | 0.0211404 | 1.0432946 | 6.786232 -instance-0 | 2.4E-4 | 0.0211404 | 1.0432946 | 6.786232 -instance-0 | 2.4E-4 | 0.0211404 | 1.0432946 | 6.786232 -instance-0 | 2.4E-4 | 0.0211404 | 1.0432946 | 6.786232 +min:double | max:double +2.17E-4 | 6.786232 ; -inlineUngroupedPercentiles -required_capability: exponential_histogram_percentiles_support + + +groupedMinMax +required_capability: exponential_histogram_minmax_support FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy") - | INLINE STATS p0 = PERCENTILE(responseTime,0), p50 = PERCENTILE(responseTime,50), p99 = PERCENTILE(responseTime, 99), p100 = PERCENTILE(responseTime,100) - | EVAL p50 = ROUND(p50, 7), p99 = ROUND(p99, 7) // rounding to avoid floating point precision issues, min and max are exact so no rounding needed - | KEEP instance, p0, p50, p99, p100 + | STATS min = MIN(responseTime), max = MAX(responseTime) BY instance + | KEEP instance, min, max | SORT instance - | LIMIT 5 ; -instance:keyword | p0:double | p50:double | p99:double | p100:double -instance-0 | 2.17E-4 | 0.0016965 | 0.9472324 | 6.786232 -instance-0 | 2.17E-4 | 0.0016965 | 0.9472324 | 6.786232 -instance-0 | 2.17E-4 | 0.0016965 | 0.9472324 | 6.786232 -instance-0 | 2.17E-4 | 0.0016965 | 0.9472324 | 6.786232 -instance-0 | 2.17E-4 | 0.0016965 | 0.9472324 | 6.786232 +instance:keyword | min:double | max:double +instance-0 | 2.4E-4 | 6.786232 +instance-1 | 2.17E-4 | 3.190723 +instance-2 | 2.2E-4 | 2.744054 ; -inlinePercentileOnEmptyHistogram -required_capability: exponential_histogram_percentiles_support + + +minMaxOnEmptyHistogram +required_capability: exponential_histogram_minmax_support FROM exp_histo_sample | WHERE instance == "dummy-empty" - | INLINE STATS p50 = PERCENTILE(responseTime,50) - | KEEP instance, p50 + | STATS min = MIN(responseTime), max = MAX(responseTime) + | KEEP min, max ; -instance:keyword | p50:double -"dummy-empty" | NULL +min:double | max:double +NULL | NULL ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index f63f15666f8a7..130bd5010f8a7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1557,6 +1557,16 @@ public enum Cap { */ EXPONENTIAL_HISTOGRAM_TOPN(EXPONENTIAL_HISTOGRAM_FEATURE_FLAG), + /** + * Support for exponential_histogram type in PERCENTILES aggregation. + */ + EXPONENTIAL_HISTOGRAM_PERCENTILES_SUPPORT(EXPONENTIAL_HISTOGRAM_FEATURE_FLAG), + + /** + * Support for exponential_histogram type in MIN/MAX aggregation. + */ + EXPONENTIAL_HISTOGRAM_MINMAX_SUPPORT(EXPONENTIAL_HISTOGRAM_FEATURE_FLAG), + /** * Create new block when filtering OrdinalBytesRefBlock */ @@ -1647,11 +1657,6 @@ public enum Cap { FULL_TEXT_FUNCTIONS_ACCEPT_NULL_FIELD, - /** - * Support for exponential_histogram type in PERCENTILES aggregation. - */ - EXPONENTIAL_HISTOGRAM_PERCENTILES_SUPPORT(EXPONENTIAL_HISTOGRAM_FEATURE_FLAG), - /** * Support for the temporary work to eventually allow FIRST to work with null and multi-value fields, among other things. */ From 5672adb4d38884402e3f9e899f638e7b67b642cf Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Fri, 14 Nov 2025 13:20:53 +0100 Subject: [PATCH 06/11] checkstyle --- .../esql/expression/function/aggregate/MinErrorTests.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinErrorTests.java index 2c52fcc68fe32..509e789b681dd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinErrorTests.java @@ -37,9 +37,8 @@ protected Matcher expectedTypeErrorMatcher(List> validPerP false, validPerPosition, signature, - ( - v, - p) -> "boolean, date, ip, string, version, aggregate_metric_double, exponential_histogram or numeric except counter types" + (v, p) -> "boolean, date, ip, string, version, aggregate_metric_double, exponential_histogram" + + " or numeric except counter types" ) ); } From f118e81f05b697ae7374a0e33c9ed2bd78876892 Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Fri, 14 Nov 2025 14:03:15 +0100 Subject: [PATCH 07/11] Checkstyle part 2 --- .../esql/expression/function/aggregate/MaxErrorTests.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxErrorTests.java index 803602647d7c1..16d0177d59057 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxErrorTests.java @@ -37,9 +37,8 @@ protected Matcher expectedTypeErrorMatcher(List> validPerP false, validPerPosition, signature, - ( - v, - p) -> "boolean, date, ip, string, version, aggregate_metric_double, exponential_histogram or numeric except counter types" + (v, p) -> "boolean, date, ip, string, version, aggregate_metric_double, " + + "exponential_histogram or numeric except counter types" ) ); } From 0f940ac17c2242ee23c680e6f2099d2f891376ea Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Fri, 14 Nov 2025 15:27:32 +0100 Subject: [PATCH 08/11] Fix csv tests sort order --- .../resources/exponential_histogram.csv-spec | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/exponential_histogram.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/exponential_histogram.csv-spec index a77fb05b14eb4..471c33062a32a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/exponential_histogram.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/exponential_histogram.csv-spec @@ -24,13 +24,14 @@ FROM exp_histo_sample | STATS min = MIN(responseTime), max = MAX(responseTime), p75 = PERCENTILE(responseTime,75) BY instance | EVAL p75 = ROUND(p75, 7) // rounding to avoid floating point precision issues | KEEP instance, min, max, p75 + | SORT instance ; instance:keyword | min:double | max:double | p75:double dummy-grouped | -100.0 | 50.0 | 8.3457089 -instance-2 | 2.2E-4 | 2.744054 | 0.0016068 instance-0 | 2.4E-4 | 6.786232 | 0.2608237 instance-1 | 2.17E-4 | 3.190723 | 0.0016068 +instance-2 | 2.2E-4 | 2.744054 | 0.0016068 ; @@ -42,24 +43,25 @@ FROM exp_histo_sample | INLINE STATS min = MIN(responseTime), max = MAX(responseTime), p75 = PERCENTILE(responseTime,75) BY instance | EVAL p75 = ROUND(p75, 7) // rounding to avoid floating point precision issues | KEEP instance, min, max, p75 + | SORT instance | Limit 15 ; instance:keyword | min:double | max:double | p75:double dummy-empty | null | null | null dummy-full | -100.0 | 50.0 | 10.6666667 +dummy-negative_only | -50.0 | -1.0 | -12.8729318 dummy-no_zero_bucket | -100.0 | 50.0 | 10.6666667 dummy-positive_only | 1.0 | 50.0 | 34.7656715 -dummy-negative_only | -50.0 | -1.0 | -12.8729318 -dummy-zero_threshold_only | null | null | null dummy-zero_count_only | 0.0 | 0.0 | 0.0 -instance-2 | 2.2E-4 | 2.744054 | 0.0016068 +dummy-zero_threshold_only | null | null | null +instance-0 | 2.4E-4 | 6.786232 | 0.2608237 +instance-0 | 2.4E-4 | 6.786232 | 0.2608237 +instance-0 | 2.4E-4 | 6.786232 | 0.2608237 +instance-0 | 2.4E-4 | 6.786232 | 0.2608237 +instance-0 | 2.4E-4 | 6.786232 | 0.2608237 instance-0 | 2.4E-4 | 6.786232 | 0.2608237 -instance-1 | 2.17E-4 | 3.190723 | 0.0016068 -instance-2 | 2.2E-4 | 2.744054 | 0.0016068 instance-0 | 2.4E-4 | 6.786232 | 0.2608237 -instance-1 | 2.17E-4 | 3.190723 | 0.0016068 -instance-2 | 2.2E-4 | 2.744054 | 0.0016068 instance-0 | 2.4E-4 | 6.786232 | 0.2608237 ; From 9de135b7f4d53d0b920b7101620ad5cdafef68a3 Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Fri, 14 Nov 2025 15:44:07 +0100 Subject: [PATCH 09/11] Fix dependent tests --- .../org/elasticsearch/xpack/esql/analysis/VerifierTests.java | 2 +- .../expression/function/aggregate/MaxOverTimeErrorTests.java | 3 ++- .../expression/function/aggregate/MinOverTimeErrorTests.java | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index b49f0465b627e..b7bde1574986b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1152,7 +1152,7 @@ public void testAggregateOnCounter() { error("FROM test | STATS min(network.bytes_in)", tsdb), equalTo( "1:19: argument of [min(network.bytes_in)] must be" - + " [boolean, date, ip, string, version, aggregate_metric_double or numeric except counter types]," + + " [boolean, date, ip, string, version, aggregate_metric_double, exponential_histogram or numeric except counter types]," + " found value [network.bytes_in] type [counter_long]" ) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxOverTimeErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxOverTimeErrorTests.java index 5071bc75239c5..9f73a45738f99 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxOverTimeErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxOverTimeErrorTests.java @@ -37,7 +37,8 @@ protected Matcher expectedTypeErrorMatcher(List> validPerP false, validPerPosition, signature, - (v, p) -> "boolean, date, ip, string, version, aggregate_metric_double or numeric except counter types" + (v, p) -> "boolean, date, ip, string, version, aggregate_metric_double, " + + "exponential_histogram or numeric except counter types" ) ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinOverTimeErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinOverTimeErrorTests.java index 109e3f0ae2e9a..f7d93a034292e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinOverTimeErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinOverTimeErrorTests.java @@ -37,7 +37,8 @@ protected Matcher expectedTypeErrorMatcher(List> validPerP false, validPerPosition, signature, - (v, p) -> "boolean, date, ip, string, version, aggregate_metric_double or numeric except counter types" + (v, p) -> "boolean, date, ip, string, version, aggregate_metric_double, " + + "exponential_histogram or numeric except counter types" ) ); } From 935063d40d910295c547661c4e02ed0420288f13 Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Mon, 17 Nov 2025 09:27:46 +0100 Subject: [PATCH 10/11] Fix checkstyle --- .../org/elasticsearch/xpack/esql/analysis/VerifierTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index b7bde1574986b..cca4cd9fb5b7f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1152,7 +1152,8 @@ public void testAggregateOnCounter() { error("FROM test | STATS min(network.bytes_in)", tsdb), equalTo( "1:19: argument of [min(network.bytes_in)] must be" - + " [boolean, date, ip, string, version, aggregate_metric_double, exponential_histogram or numeric except counter types]," + + " [boolean, date, ip, string, version, aggregate_metric_double," + + " exponential_histogram or numeric except counter types]," + " found value [network.bytes_in] type [counter_long]" ) ); From 1793d10388a02dfb19818e2b6071f952507888f8 Mon Sep 17 00:00:00 2001 From: Jonas Kunz Date: Mon, 17 Nov 2025 10:38:24 +0100 Subject: [PATCH 11/11] fix VerifierTests --- .../org/elasticsearch/xpack/esql/analysis/VerifierTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index cca4cd9fb5b7f..1de00620b898b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1162,7 +1162,8 @@ public void testAggregateOnCounter() { error("FROM test | STATS max(network.bytes_in)", tsdb), equalTo( "1:19: argument of [max(network.bytes_in)] must be" - + " [boolean, date, ip, string, version, aggregate_metric_double or numeric except counter types]," + + " [boolean, date, ip, string, version, aggregate_metric_double, exponential_histogram" + + " or numeric except counter types]," + " found value [network.bytes_in] type [counter_long]" ) );