From d7f08bdfa5fef1844f094add2a37b6c62505485c Mon Sep 17 00:00:00 2001 From: Pablo Date: Thu, 16 Oct 2025 11:06:22 -0700 Subject: [PATCH 01/24] Stddev over time --- .../main/resources/k8s-timeseries.csv-spec | 23 ++++ .../function/EsqlFunctionRegistry.java | 2 + .../aggregate/AggregateWritables.java | 1 + .../function/aggregate/StdDevOverTime.java | 100 ++++++++++++++++++ .../aggregate/StdDevOverTimeTests.java | 34 ++++++ 5 files changed, 160 insertions(+) create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTime.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTimeTests.java diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec index 4109312b16f1c..d923a5883726c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec @@ -593,3 +593,26 @@ max_cost:integer | pod:keyword | time_bucket:datetime | max_cluster_cost:int 1209 | three | 2024-05-10T00:00:00.000Z | 1209 | staging 973 | two | 2024-05-10T00:00:00.000Z | 1209 | staging ; + +max_of_stddev_over_time +required_capability: ts_command_v0 +// tag::stddev_over_time[] +TS k8s +| STATS max_stddev_cost=MAX(STDDEV_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) +// end::stddev_over_time[] +| SORT max_stddev_cost DESC, time_bucket DESC, cluster | LIMIT 10; + +// tag::stddev_over_time-result[] +max_stddev_cost:double | cluster:keyword | time_bucket:datetime +5.4375 | staging | 2024-05-10T00:03:00.000Z +5.1875 | staging | 2024-05-10T00:09:00.000Z +4.097763617714749 | qa | 2024-05-10T00:18:00.000Z +4.0 | qa | 2024-05-10T00:21:00.000Z +3.9375 | staging | 2024-05-10T00:20:00.000Z +3.9335297443898907 | prod | 2024-05-10T00:18:00.000Z +// end::stddev_over_time-result[] +3.8944404818493075 | qa | 2024-05-10T00:08:00.000Z +3.8823854350987186 | qa | 2024-05-10T00:17:00.000Z +3.835597157621686 | staging | 2024-05-10T00:13:00.000Z +3.8125 | staging | 2024-05-10T00:12:00.000Z +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 2f4d72338b4fc..17a560cfea7ae 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -49,6 +49,7 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialCentroid; import org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialExtent; import org.elasticsearch.xpack.esql.expression.function.aggregate.StdDev; +import org.elasticsearch.xpack.esql.expression.function.aggregate.StdDevOverTime; import org.elasticsearch.xpack.esql.expression.function.aggregate.Sum; import org.elasticsearch.xpack.esql.expression.function.aggregate.SumOverTime; import org.elasticsearch.xpack.esql.expression.function.aggregate.Top; @@ -527,6 +528,7 @@ private static FunctionDefinition[][] functions() { def(MaxOverTime.class, uni(MaxOverTime::new), "max_over_time"), def(MinOverTime.class, uni(MinOverTime::new), "min_over_time"), def(SumOverTime.class, uni(SumOverTime::new), "sum_over_time"), + def(StdDevOverTime.class, uni(StdDevOverTime::new), "stddev_over_time"), def(CountOverTime.class, uni(CountOverTime::new), "count_over_time"), def(CountDistinctOverTime.class, bi(CountDistinctOverTime::new), "count_distinct_over_time"), def(PresentOverTime.class, uni(PresentOverTime::new), "present_over_time"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateWritables.java index d604a72dce08c..144d54887b565 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateWritables.java @@ -39,6 +39,7 @@ public static List getNamedWriteables() { MinOverTime.ENTRY, MaxOverTime.ENTRY, AvgOverTime.ENTRY, + StdDevOverTime.ENTRY, Last.ENTRY, LastOverTime.ENTRY, FirstOverTime.ENTRY, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTime.java new file mode 100644 index 0000000000000..7b0b0d8f65e8d --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTime.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.aggregate; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesTo; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesToLifecycle; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.FunctionType; +import org.elasticsearch.xpack.esql.expression.function.Param; + +import java.io.IOException; +import java.util.List; + +import static java.util.Collections.emptyList; + +/** + * Similar to {@link StdDev}, but it is used to calculate the standard deviation over a time series of values from the given field. + */ +public class StdDevOverTime extends TimeSeriesAggregateFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "StdDevOverTime", + StdDevOverTime::new + ); + + @FunctionInfo( + returnType = "double", + description = "Calculates the population standard deviation over time of a numeric field.", + type = FunctionType.TIME_SERIES_AGGREGATE, + appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0") }, + preview = true, + examples = { @Example(file = "k8s-timeseries", tag = "stddev_over_time") } + ) + public StdDevOverTime( + Source source, + @Param( + name = "number", + type = { "aggregate_metric_double", "double", "integer", "long" }, + description = "Expression that outputs values to calculate the standard deviation of." + ) Expression field + ) { + this(source, field, Literal.TRUE); + } + + public StdDevOverTime(Source source, Expression field, Expression filter) { + super(source, field, filter, emptyList()); + } + + private StdDevOverTime(StreamInput in) throws IOException { + super(in); + } + + @Override + protected TypeResolution resolveType() { + return perTimeSeriesAggregation().resolveType(); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public DataType dataType() { + return perTimeSeriesAggregation().dataType(); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, StdDevOverTime::new, field(), filter()); + } + + @Override + public StdDevOverTime replaceChildren(List newChildren) { + return new StdDevOverTime(source(), newChildren.get(0), newChildren.get(1)); + } + + @Override + public StdDevOverTime withFilter(Expression filter) { + return new StdDevOverTime(source(), field(), filter); + } + + @Override + public AggregateFunction perTimeSeriesAggregation() { + return new StdDev(source(), field(), filter()); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTimeTests.java new file mode 100644 index 0000000000000..cf537ddb6c27d --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTimeTests.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.aggregate; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.List; +import java.util.function.Supplier; + +public class StdDevOverTimeTests extends AbstractFunctionTestCase { + public StdDevOverTimeTests(Supplier testCaseSupplier) { + testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + return StdDevTests.parameters(); + } + + @Override + protected Expression build(Source source, List args) { + return new StdDevOverTime(source, args.get(0)); + } +} From 83db147105434f45b4844dd67f5c037a8323282c Mon Sep 17 00:00:00 2001 From: Pablo Date: Thu, 16 Oct 2025 15:19:04 -0700 Subject: [PATCH 02/24] preparing variance changes --- .../aggregation/StdDevDoubleAggregator.java | 20 +++++++++---------- .../aggregation/StdDevFloatAggregator.java | 20 +++++++++---------- .../aggregation/StdDevIntAggregator.java | 20 +++++++++---------- .../aggregation/StdDevLongAggregator.java | 20 +++++++++---------- .../StdDevDoubleAggregatorFunction.java | 4 ++-- ...tdDevDoubleGroupingAggregatorFunction.java | 4 ++-- .../StdDevFloatAggregatorFunction.java | 4 ++-- ...StdDevFloatGroupingAggregatorFunction.java | 4 ++-- .../StdDevIntAggregatorFunction.java | 4 ++-- .../StdDevIntGroupingAggregatorFunction.java | 4 ++-- .../StdDevLongAggregatorFunction.java | 4 ++-- .../StdDevLongGroupingAggregatorFunction.java | 4 ++-- ...{StdDevStates.java => VarianceStates.java} | 18 +++++++++-------- .../compute/aggregation/WelfordAlgorithm.java | 17 +++++++++++++--- 14 files changed, 80 insertions(+), 67 deletions(-) rename x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/{StdDevStates.java => VarianceStates.java} (94%) diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevDoubleAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevDoubleAggregator.java index 8cf0809164aa1..ef544fabcf5d0 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevDoubleAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevDoubleAggregator.java @@ -28,35 +28,35 @@ @GroupingAggregator public class StdDevDoubleAggregator { - public static StdDevStates.SingleState initSingle() { - return new StdDevStates.SingleState(); + public static VarianceStates.SingleState initSingle() { + return new VarianceStates.SingleState(); } - public static void combine(StdDevStates.SingleState state, double value) { + public static void combine(VarianceStates.SingleState state, double value) { state.add(value); } - public static void combineIntermediate(StdDevStates.SingleState state, double mean, double m2, long count) { + public static void combineIntermediate(VarianceStates.SingleState state, double mean, double m2, long count) { state.combine(mean, m2, count); } - public static Block evaluateFinal(StdDevStates.SingleState state, DriverContext driverContext) { + public static Block evaluateFinal(VarianceStates.SingleState state, DriverContext driverContext) { return state.evaluateFinal(driverContext); } - public static StdDevStates.GroupingState initGrouping(BigArrays bigArrays) { - return new StdDevStates.GroupingState(bigArrays); + public static VarianceStates.GroupingState initGrouping(BigArrays bigArrays) { + return new VarianceStates.GroupingState(bigArrays); } - public static void combine(StdDevStates.GroupingState current, int groupId, double value) { + public static void combine(VarianceStates.GroupingState current, int groupId, double value) { current.add(groupId, value); } - public static void combineIntermediate(StdDevStates.GroupingState state, int groupId, double mean, double m2, long count) { + public static void combineIntermediate(VarianceStates.GroupingState state, int groupId, double mean, double m2, long count) { state.combine(groupId, mean, m2, count); } - public static Block evaluateFinal(StdDevStates.GroupingState state, IntVector selected, GroupingAggregatorEvaluationContext ctx) { + public static Block evaluateFinal(VarianceStates.GroupingState state, IntVector selected, GroupingAggregatorEvaluationContext ctx) { return state.evaluateFinal(selected, ctx.driverContext()); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevFloatAggregator.java index 7fd851a16ab8f..0bdb4e6ebb1eb 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevFloatAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevFloatAggregator.java @@ -28,35 +28,35 @@ @GroupingAggregator public class StdDevFloatAggregator { - public static StdDevStates.SingleState initSingle() { - return new StdDevStates.SingleState(); + public static VarianceStates.SingleState initSingle() { + return new VarianceStates.SingleState(); } - public static void combine(StdDevStates.SingleState state, float value) { + public static void combine(VarianceStates.SingleState state, float value) { state.add(value); } - public static void combineIntermediate(StdDevStates.SingleState state, double mean, double m2, long count) { + public static void combineIntermediate(VarianceStates.SingleState state, double mean, double m2, long count) { state.combine(mean, m2, count); } - public static Block evaluateFinal(StdDevStates.SingleState state, DriverContext driverContext) { + public static Block evaluateFinal(VarianceStates.SingleState state, DriverContext driverContext) { return state.evaluateFinal(driverContext); } - public static StdDevStates.GroupingState initGrouping(BigArrays bigArrays) { - return new StdDevStates.GroupingState(bigArrays); + public static VarianceStates.GroupingState initGrouping(BigArrays bigArrays) { + return new VarianceStates.GroupingState(bigArrays); } - public static void combine(StdDevStates.GroupingState current, int groupId, float value) { + public static void combine(VarianceStates.GroupingState current, int groupId, float value) { current.add(groupId, value); } - public static void combineIntermediate(StdDevStates.GroupingState state, int groupId, double mean, double m2, long count) { + public static void combineIntermediate(VarianceStates.GroupingState state, int groupId, double mean, double m2, long count) { state.combine(groupId, mean, m2, count); } - public static Block evaluateFinal(StdDevStates.GroupingState state, IntVector selected, GroupingAggregatorEvaluationContext ctx) { + public static Block evaluateFinal(VarianceStates.GroupingState state, IntVector selected, GroupingAggregatorEvaluationContext ctx) { return state.evaluateFinal(selected, ctx.driverContext()); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevIntAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevIntAggregator.java index 30adf0e28b1b4..419f778581eb3 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevIntAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevIntAggregator.java @@ -28,35 +28,35 @@ @GroupingAggregator public class StdDevIntAggregator { - public static StdDevStates.SingleState initSingle() { - return new StdDevStates.SingleState(); + public static VarianceStates.SingleState initSingle() { + return new VarianceStates.SingleState(); } - public static void combine(StdDevStates.SingleState state, int value) { + public static void combine(VarianceStates.SingleState state, int value) { state.add(value); } - public static void combineIntermediate(StdDevStates.SingleState state, double mean, double m2, long count) { + public static void combineIntermediate(VarianceStates.SingleState state, double mean, double m2, long count) { state.combine(mean, m2, count); } - public static Block evaluateFinal(StdDevStates.SingleState state, DriverContext driverContext) { + public static Block evaluateFinal(VarianceStates.SingleState state, DriverContext driverContext) { return state.evaluateFinal(driverContext); } - public static StdDevStates.GroupingState initGrouping(BigArrays bigArrays) { - return new StdDevStates.GroupingState(bigArrays); + public static VarianceStates.GroupingState initGrouping(BigArrays bigArrays) { + return new VarianceStates.GroupingState(bigArrays); } - public static void combine(StdDevStates.GroupingState current, int groupId, int value) { + public static void combine(VarianceStates.GroupingState current, int groupId, int value) { current.add(groupId, value); } - public static void combineIntermediate(StdDevStates.GroupingState state, int groupId, double mean, double m2, long count) { + public static void combineIntermediate(VarianceStates.GroupingState state, int groupId, double mean, double m2, long count) { state.combine(groupId, mean, m2, count); } - public static Block evaluateFinal(StdDevStates.GroupingState state, IntVector selected, GroupingAggregatorEvaluationContext ctx) { + public static Block evaluateFinal(VarianceStates.GroupingState state, IntVector selected, GroupingAggregatorEvaluationContext ctx) { return state.evaluateFinal(selected, ctx.driverContext()); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevLongAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevLongAggregator.java index 4f6b5008d79eb..20d8fc6a49077 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevLongAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevLongAggregator.java @@ -28,35 +28,35 @@ @GroupingAggregator public class StdDevLongAggregator { - public static StdDevStates.SingleState initSingle() { - return new StdDevStates.SingleState(); + public static VarianceStates.SingleState initSingle() { + return new VarianceStates.SingleState(); } - public static void combine(StdDevStates.SingleState state, long value) { + public static void combine(VarianceStates.SingleState state, long value) { state.add(value); } - public static void combineIntermediate(StdDevStates.SingleState state, double mean, double m2, long count) { + public static void combineIntermediate(VarianceStates.SingleState state, double mean, double m2, long count) { state.combine(mean, m2, count); } - public static Block evaluateFinal(StdDevStates.SingleState state, DriverContext driverContext) { + public static Block evaluateFinal(VarianceStates.SingleState state, DriverContext driverContext) { return state.evaluateFinal(driverContext); } - public static StdDevStates.GroupingState initGrouping(BigArrays bigArrays) { - return new StdDevStates.GroupingState(bigArrays); + public static VarianceStates.GroupingState initGrouping(BigArrays bigArrays) { + return new VarianceStates.GroupingState(bigArrays); } - public static void combine(StdDevStates.GroupingState current, int groupId, long value) { + public static void combine(VarianceStates.GroupingState current, int groupId, long value) { current.add(groupId, value); } - public static void combineIntermediate(StdDevStates.GroupingState state, int groupId, double mean, double m2, long count) { + public static void combineIntermediate(VarianceStates.GroupingState state, int groupId, double mean, double m2, long count) { state.combine(groupId, mean, m2, count); } - public static Block evaluateFinal(StdDevStates.GroupingState state, IntVector selected, GroupingAggregatorEvaluationContext ctx) { + public static Block evaluateFinal(VarianceStates.GroupingState state, IntVector selected, GroupingAggregatorEvaluationContext ctx) { return state.evaluateFinal(selected, ctx.driverContext()); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleAggregatorFunction.java index 208a293f7aeb8..8810e4be8590b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleAggregatorFunction.java @@ -31,12 +31,12 @@ public final class StdDevDoubleAggregatorFunction implements AggregatorFunction private final DriverContext driverContext; - private final StdDevStates.SingleState state; + private final VarianceStates.SingleState state; private final List channels; public StdDevDoubleAggregatorFunction(DriverContext driverContext, List channels, - StdDevStates.SingleState state) { + VarianceStates.SingleState state) { this.driverContext = driverContext; this.channels = channels; this.state = state; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleGroupingAggregatorFunction.java index b4337f1030adb..2a5aabbd872c9 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleGroupingAggregatorFunction.java @@ -31,14 +31,14 @@ public final class StdDevDoubleGroupingAggregatorFunction implements GroupingAgg new IntermediateStateDesc("m2", ElementType.DOUBLE), new IntermediateStateDesc("count", ElementType.LONG) ); - private final StdDevStates.GroupingState state; + private final VarianceStates.GroupingState state; private final List channels; private final DriverContext driverContext; public StdDevDoubleGroupingAggregatorFunction(List channels, - StdDevStates.GroupingState state, DriverContext driverContext) { + VarianceStates.GroupingState state, DriverContext driverContext) { this.channels = channels; this.state = state; this.driverContext = driverContext; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatAggregatorFunction.java index c82d0a3ab01d3..546f5beae15f7 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatAggregatorFunction.java @@ -33,12 +33,12 @@ public final class StdDevFloatAggregatorFunction implements AggregatorFunction { private final DriverContext driverContext; - private final StdDevStates.SingleState state; + private final VarianceStates.SingleState state; private final List channels; public StdDevFloatAggregatorFunction(DriverContext driverContext, List channels, - StdDevStates.SingleState state) { + VarianceStates.SingleState state) { this.driverContext = driverContext; this.channels = channels; this.state = state; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatGroupingAggregatorFunction.java index 7e513e1f9dc35..808c026a5a646 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatGroupingAggregatorFunction.java @@ -33,14 +33,14 @@ public final class StdDevFloatGroupingAggregatorFunction implements GroupingAggr new IntermediateStateDesc("m2", ElementType.DOUBLE), new IntermediateStateDesc("count", ElementType.LONG) ); - private final StdDevStates.GroupingState state; + private final VarianceStates.GroupingState state; private final List channels; private final DriverContext driverContext; public StdDevFloatGroupingAggregatorFunction(List channels, - StdDevStates.GroupingState state, DriverContext driverContext) { + VarianceStates.GroupingState state, DriverContext driverContext) { this.channels = channels; this.state = state; this.driverContext = driverContext; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntAggregatorFunction.java index 6b9f7f4e561ac..1aa0ee823eca8 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntAggregatorFunction.java @@ -33,12 +33,12 @@ public final class StdDevIntAggregatorFunction implements AggregatorFunction { private final DriverContext driverContext; - private final StdDevStates.SingleState state; + private final VarianceStates.SingleState state; private final List channels; public StdDevIntAggregatorFunction(DriverContext driverContext, List channels, - StdDevStates.SingleState state) { + VarianceStates.SingleState state) { this.driverContext = driverContext; this.channels = channels; this.state = state; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntGroupingAggregatorFunction.java index 1145ccfd4bce3..858ce287ae6d6 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntGroupingAggregatorFunction.java @@ -32,14 +32,14 @@ public final class StdDevIntGroupingAggregatorFunction implements GroupingAggreg new IntermediateStateDesc("m2", ElementType.DOUBLE), new IntermediateStateDesc("count", ElementType.LONG) ); - private final StdDevStates.GroupingState state; + private final VarianceStates.GroupingState state; private final List channels; private final DriverContext driverContext; public StdDevIntGroupingAggregatorFunction(List channels, - StdDevStates.GroupingState state, DriverContext driverContext) { + VarianceStates.GroupingState state, DriverContext driverContext) { this.channels = channels; this.state = state; this.driverContext = driverContext; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongAggregatorFunction.java index 0e53e5f2d461e..e5563c380bfdd 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongAggregatorFunction.java @@ -31,12 +31,12 @@ public final class StdDevLongAggregatorFunction implements AggregatorFunction { private final DriverContext driverContext; - private final StdDevStates.SingleState state; + private final VarianceStates.SingleState state; private final List channels; public StdDevLongAggregatorFunction(DriverContext driverContext, List channels, - StdDevStates.SingleState state) { + VarianceStates.SingleState state) { this.driverContext = driverContext; this.channels = channels; this.state = state; diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongGroupingAggregatorFunction.java index 5e2dde7ea1fde..64f0ec2571687 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongGroupingAggregatorFunction.java @@ -31,14 +31,14 @@ public final class StdDevLongGroupingAggregatorFunction implements GroupingAggre new IntermediateStateDesc("m2", ElementType.DOUBLE), new IntermediateStateDesc("count", ElementType.LONG) ); - private final StdDevStates.GroupingState state; + private final VarianceStates.GroupingState state; private final List channels; private final DriverContext driverContext; public StdDevLongGroupingAggregatorFunction(List channels, - StdDevStates.GroupingState state, DriverContext driverContext) { + VarianceStates.GroupingState state, DriverContext driverContext) { this.channels = channels; this.state = state; this.driverContext = driverContext; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/StdDevStates.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/VarianceStates.java similarity index 94% rename from x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/StdDevStates.java rename to x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/VarianceStates.java index 5b48498d83294..2cf0921205f4f 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/StdDevStates.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/VarianceStates.java @@ -16,20 +16,20 @@ import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.core.Releasables; -public final class StdDevStates { +public final class VarianceStates { - private StdDevStates() {} + private VarianceStates() {} static final class SingleState implements AggregatorState { private final WelfordAlgorithm welfordAlgorithm; SingleState() { - this(0, 0, 0); + this(0, 0, 0, true); } - SingleState(double mean, double m2, long count) { - this.welfordAlgorithm = new WelfordAlgorithm(mean, m2, count); + SingleState(double mean, double m2, long count, boolean stdDev) { + this.welfordAlgorithm = new WelfordAlgorithm(mean, m2, count, stdDev); } public void add(long value) { @@ -90,10 +90,12 @@ static final class GroupingState implements GroupingAggregatorState { private ObjectArray states; private final BigArrays bigArrays; + private final boolean stdDev; - GroupingState(BigArrays bigArrays) { + GroupingState(BigArrays bigArrays, boolean stdDev) { this.states = bigArrays.newObjectArray(1); this.bigArrays = bigArrays; + this.stdDev = stdDev; } WelfordAlgorithm getOrNull(int position) { @@ -115,7 +117,7 @@ public void combine(int groupId, double meanValue, double m2Value, long countVal ensureCapacity(groupId); var state = states.get(groupId); if (state == null) { - state = new WelfordAlgorithm(meanValue, m2Value, countValue); + state = new WelfordAlgorithm(meanValue, m2Value, countValue, stdDev); states.set(groupId, state); } else { state.add(meanValue, m2Value, countValue); @@ -126,7 +128,7 @@ public WelfordAlgorithm getOrSet(int groupId) { ensureCapacity(groupId); var state = states.get(groupId); if (state == null) { - state = new WelfordAlgorithm(); + state = new WelfordAlgorithm(stdDev); states.set(groupId, state); } return state; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/WelfordAlgorithm.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/WelfordAlgorithm.java index 8ccb985507247..d1dce9e93aaf6 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/WelfordAlgorithm.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/WelfordAlgorithm.java @@ -19,6 +19,8 @@ public final class WelfordAlgorithm { private double mean; private double m2; private long count; + // If true, compute standard deviation, otherwise compute variance + private final boolean stdDev; public double mean() { return mean; @@ -33,13 +35,18 @@ public long count() { } public WelfordAlgorithm() { - this(0, 0, 0); + this(0, 0, 0, true); } - public WelfordAlgorithm(double mean, double m2, long count) { + public WelfordAlgorithm(boolean stdDev) { + this(0, 0, 0, stdDev); + } + + public WelfordAlgorithm(double mean, double m2, long count, boolean stdDev) { this.mean = mean; this.m2 = m2; this.count = count; + this.stdDev = stdDev; } public void add(int value) { @@ -74,6 +81,10 @@ public void add(double meanValue, double m2Value, long countValue) { } public double evaluate() { - return count < 2 ? 0 : Math.sqrt(m2 / count); + if (!stdDev) { + return count < 2 ? 0 : m2 / count; + } else { + return count < 2 ? 0 : Math.sqrt(m2 / count); + } } } From fb6913b63de029c40b472c060ffeeeba3dc5db84 Mon Sep 17 00:00:00 2001 From: Pablo Date: Thu, 16 Oct 2025 15:50:38 -0700 Subject: [PATCH 03/24] Adding stdvar as well, and an esql capability to fix other tests --- .../aggregation/StdDevDoubleAggregator.java | 8 +- .../aggregation/StdDevFloatAggregator.java | 8 +- .../aggregation/StdDevIntAggregator.java | 8 +- .../aggregation/StdDevLongAggregator.java | 8 +- .../StdDevDoubleAggregatorFunction.java | 9 +- ...tdDevDoubleAggregatorFunctionSupplier.java | 9 +- ...tdDevDoubleGroupingAggregatorFunction.java | 9 +- .../StdDevFloatAggregatorFunction.java | 9 +- ...StdDevFloatAggregatorFunctionSupplier.java | 9 +- ...StdDevFloatGroupingAggregatorFunction.java | 9 +- .../StdDevIntAggregatorFunction.java | 9 +- .../StdDevIntAggregatorFunctionSupplier.java | 9 +- .../StdDevIntGroupingAggregatorFunction.java | 9 +- .../StdDevLongAggregatorFunction.java | 9 +- .../StdDevLongAggregatorFunctionSupplier.java | 9 +- .../StdDevLongGroupingAggregatorFunction.java | 9 +- .../compute/aggregation/VarianceStates.java | 4 +- .../aggregation/X-StdDevAggregator.java.st | 20 ++-- .../main/resources/k8s-timeseries.csv-spec | 49 +++++++++ .../xpack/esql/action/EsqlCapabilities.java | 2 +- .../function/EsqlFunctionRegistry.java | 4 + .../aggregate/AggregateWritables.java | 2 + .../expression/function/aggregate/StdDev.java | 6 +- .../expression/function/aggregate/StdVar.java | 103 ++++++++++++++++++ .../function/aggregate/StdVarOverTime.java | 100 +++++++++++++++++ .../aggregate/StdVarOverTimeTests.java | 34 ++++++ .../function/aggregate/StdVarTests.java | 73 +++++++++++++ 27 files changed, 469 insertions(+), 68 deletions(-) create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVar.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTime.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTimeTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevDoubleAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevDoubleAggregator.java index ef544fabcf5d0..cb2bc98c79e65 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevDoubleAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevDoubleAggregator.java @@ -28,8 +28,8 @@ @GroupingAggregator public class StdDevDoubleAggregator { - public static VarianceStates.SingleState initSingle() { - return new VarianceStates.SingleState(); + public static VarianceStates.SingleState initSingle(boolean stdDev) { + return new VarianceStates.SingleState(stdDev); } public static void combine(VarianceStates.SingleState state, double value) { @@ -44,8 +44,8 @@ public static Block evaluateFinal(VarianceStates.SingleState state, DriverContex return state.evaluateFinal(driverContext); } - public static VarianceStates.GroupingState initGrouping(BigArrays bigArrays) { - return new VarianceStates.GroupingState(bigArrays); + public static VarianceStates.GroupingState initGrouping(BigArrays bigArrays, boolean stdDev) { + return new VarianceStates.GroupingState(bigArrays, stdDev); } public static void combine(VarianceStates.GroupingState current, int groupId, double value) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevFloatAggregator.java index 0bdb4e6ebb1eb..2179e2053513e 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevFloatAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevFloatAggregator.java @@ -28,8 +28,8 @@ @GroupingAggregator public class StdDevFloatAggregator { - public static VarianceStates.SingleState initSingle() { - return new VarianceStates.SingleState(); + public static VarianceStates.SingleState initSingle(boolean stdDev) { + return new VarianceStates.SingleState(stdDev); } public static void combine(VarianceStates.SingleState state, float value) { @@ -44,8 +44,8 @@ public static Block evaluateFinal(VarianceStates.SingleState state, DriverContex return state.evaluateFinal(driverContext); } - public static VarianceStates.GroupingState initGrouping(BigArrays bigArrays) { - return new VarianceStates.GroupingState(bigArrays); + public static VarianceStates.GroupingState initGrouping(BigArrays bigArrays, boolean stdDev) { + return new VarianceStates.GroupingState(bigArrays, stdDev); } public static void combine(VarianceStates.GroupingState current, int groupId, float value) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevIntAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevIntAggregator.java index 419f778581eb3..851b611ed7f29 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevIntAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevIntAggregator.java @@ -28,8 +28,8 @@ @GroupingAggregator public class StdDevIntAggregator { - public static VarianceStates.SingleState initSingle() { - return new VarianceStates.SingleState(); + public static VarianceStates.SingleState initSingle(boolean stdDev) { + return new VarianceStates.SingleState(stdDev); } public static void combine(VarianceStates.SingleState state, int value) { @@ -44,8 +44,8 @@ public static Block evaluateFinal(VarianceStates.SingleState state, DriverContex return state.evaluateFinal(driverContext); } - public static VarianceStates.GroupingState initGrouping(BigArrays bigArrays) { - return new VarianceStates.GroupingState(bigArrays); + public static VarianceStates.GroupingState initGrouping(BigArrays bigArrays, boolean stdDev) { + return new VarianceStates.GroupingState(bigArrays, stdDev); } public static void combine(VarianceStates.GroupingState current, int groupId, int value) { diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevLongAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevLongAggregator.java index 20d8fc6a49077..1c1d90e187000 100644 --- a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevLongAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/StdDevLongAggregator.java @@ -28,8 +28,8 @@ @GroupingAggregator public class StdDevLongAggregator { - public static VarianceStates.SingleState initSingle() { - return new VarianceStates.SingleState(); + public static VarianceStates.SingleState initSingle(boolean stdDev) { + return new VarianceStates.SingleState(stdDev); } public static void combine(VarianceStates.SingleState state, long value) { @@ -44,8 +44,8 @@ public static Block evaluateFinal(VarianceStates.SingleState state, DriverContex return state.evaluateFinal(driverContext); } - public static VarianceStates.GroupingState initGrouping(BigArrays bigArrays) { - return new VarianceStates.GroupingState(bigArrays); + public static VarianceStates.GroupingState initGrouping(BigArrays bigArrays, boolean stdDev) { + return new VarianceStates.GroupingState(bigArrays, stdDev); } public static void combine(VarianceStates.GroupingState current, int groupId, long value) { diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleAggregatorFunction.java index 8810e4be8590b..727b0a6226f31 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleAggregatorFunction.java @@ -35,16 +35,19 @@ public final class StdDevDoubleAggregatorFunction implements AggregatorFunction private final List channels; + private final boolean stdDev; + public StdDevDoubleAggregatorFunction(DriverContext driverContext, List channels, - VarianceStates.SingleState state) { + VarianceStates.SingleState state, boolean stdDev) { this.driverContext = driverContext; this.channels = channels; this.state = state; + this.stdDev = stdDev; } public static StdDevDoubleAggregatorFunction create(DriverContext driverContext, - List channels) { - return new StdDevDoubleAggregatorFunction(driverContext, channels, StdDevDoubleAggregator.initSingle()); + List channels, boolean stdDev) { + return new StdDevDoubleAggregatorFunction(driverContext, channels, StdDevDoubleAggregator.initSingle(stdDev), stdDev); } public static List intermediateStateDesc() { diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleAggregatorFunctionSupplier.java index 5310a11c1fddb..ada74a2a5fc1e 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleAggregatorFunctionSupplier.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleAggregatorFunctionSupplier.java @@ -15,7 +15,10 @@ * This class is generated. Edit {@code AggregatorFunctionSupplierImplementer} instead. */ public final class StdDevDoubleAggregatorFunctionSupplier implements AggregatorFunctionSupplier { - public StdDevDoubleAggregatorFunctionSupplier() { + private final boolean stdDev; + + public StdDevDoubleAggregatorFunctionSupplier(boolean stdDev) { + this.stdDev = stdDev; } @Override @@ -31,13 +34,13 @@ public List groupingIntermediateStateDesc() { @Override public StdDevDoubleAggregatorFunction aggregator(DriverContext driverContext, List channels) { - return StdDevDoubleAggregatorFunction.create(driverContext, channels); + return StdDevDoubleAggregatorFunction.create(driverContext, channels, stdDev); } @Override public StdDevDoubleGroupingAggregatorFunction groupingAggregator(DriverContext driverContext, List channels) { - return StdDevDoubleGroupingAggregatorFunction.create(channels, driverContext); + return StdDevDoubleGroupingAggregatorFunction.create(channels, driverContext, stdDev); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleGroupingAggregatorFunction.java index 2a5aabbd872c9..1738282c7ee0b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevDoubleGroupingAggregatorFunction.java @@ -37,16 +37,19 @@ public final class StdDevDoubleGroupingAggregatorFunction implements GroupingAgg private final DriverContext driverContext; + private final boolean stdDev; + public StdDevDoubleGroupingAggregatorFunction(List channels, - VarianceStates.GroupingState state, DriverContext driverContext) { + VarianceStates.GroupingState state, DriverContext driverContext, boolean stdDev) { this.channels = channels; this.state = state; this.driverContext = driverContext; + this.stdDev = stdDev; } public static StdDevDoubleGroupingAggregatorFunction create(List channels, - DriverContext driverContext) { - return new StdDevDoubleGroupingAggregatorFunction(channels, StdDevDoubleAggregator.initGrouping(driverContext.bigArrays()), driverContext); + DriverContext driverContext, boolean stdDev) { + return new StdDevDoubleGroupingAggregatorFunction(channels, StdDevDoubleAggregator.initGrouping(driverContext.bigArrays(), stdDev), driverContext, stdDev); } public static List intermediateStateDesc() { diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatAggregatorFunction.java index 546f5beae15f7..0dc56fc22040c 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatAggregatorFunction.java @@ -37,16 +37,19 @@ public final class StdDevFloatAggregatorFunction implements AggregatorFunction { private final List channels; + private final boolean stdDev; + public StdDevFloatAggregatorFunction(DriverContext driverContext, List channels, - VarianceStates.SingleState state) { + VarianceStates.SingleState state, boolean stdDev) { this.driverContext = driverContext; this.channels = channels; this.state = state; + this.stdDev = stdDev; } public static StdDevFloatAggregatorFunction create(DriverContext driverContext, - List channels) { - return new StdDevFloatAggregatorFunction(driverContext, channels, StdDevFloatAggregator.initSingle()); + List channels, boolean stdDev) { + return new StdDevFloatAggregatorFunction(driverContext, channels, StdDevFloatAggregator.initSingle(stdDev), stdDev); } public static List intermediateStateDesc() { diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatAggregatorFunctionSupplier.java index 52ffb0f5d580d..a233f89757350 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatAggregatorFunctionSupplier.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatAggregatorFunctionSupplier.java @@ -15,7 +15,10 @@ * This class is generated. Edit {@code AggregatorFunctionSupplierImplementer} instead. */ public final class StdDevFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { - public StdDevFloatAggregatorFunctionSupplier() { + private final boolean stdDev; + + public StdDevFloatAggregatorFunctionSupplier(boolean stdDev) { + this.stdDev = stdDev; } @Override @@ -31,13 +34,13 @@ public List groupingIntermediateStateDesc() { @Override public StdDevFloatAggregatorFunction aggregator(DriverContext driverContext, List channels) { - return StdDevFloatAggregatorFunction.create(driverContext, channels); + return StdDevFloatAggregatorFunction.create(driverContext, channels, stdDev); } @Override public StdDevFloatGroupingAggregatorFunction groupingAggregator(DriverContext driverContext, List channels) { - return StdDevFloatGroupingAggregatorFunction.create(channels, driverContext); + return StdDevFloatGroupingAggregatorFunction.create(channels, driverContext, stdDev); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatGroupingAggregatorFunction.java index 808c026a5a646..5ee9758c98d45 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevFloatGroupingAggregatorFunction.java @@ -39,16 +39,19 @@ public final class StdDevFloatGroupingAggregatorFunction implements GroupingAggr private final DriverContext driverContext; + private final boolean stdDev; + public StdDevFloatGroupingAggregatorFunction(List channels, - VarianceStates.GroupingState state, DriverContext driverContext) { + VarianceStates.GroupingState state, DriverContext driverContext, boolean stdDev) { this.channels = channels; this.state = state; this.driverContext = driverContext; + this.stdDev = stdDev; } public static StdDevFloatGroupingAggregatorFunction create(List channels, - DriverContext driverContext) { - return new StdDevFloatGroupingAggregatorFunction(channels, StdDevFloatAggregator.initGrouping(driverContext.bigArrays()), driverContext); + DriverContext driverContext, boolean stdDev) { + return new StdDevFloatGroupingAggregatorFunction(channels, StdDevFloatAggregator.initGrouping(driverContext.bigArrays(), stdDev), driverContext, stdDev); } public static List intermediateStateDesc() { diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntAggregatorFunction.java index 1aa0ee823eca8..0a59974587b59 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntAggregatorFunction.java @@ -37,16 +37,19 @@ public final class StdDevIntAggregatorFunction implements AggregatorFunction { private final List channels; + private final boolean stdDev; + public StdDevIntAggregatorFunction(DriverContext driverContext, List channels, - VarianceStates.SingleState state) { + VarianceStates.SingleState state, boolean stdDev) { this.driverContext = driverContext; this.channels = channels; this.state = state; + this.stdDev = stdDev; } public static StdDevIntAggregatorFunction create(DriverContext driverContext, - List channels) { - return new StdDevIntAggregatorFunction(driverContext, channels, StdDevIntAggregator.initSingle()); + List channels, boolean stdDev) { + return new StdDevIntAggregatorFunction(driverContext, channels, StdDevIntAggregator.initSingle(stdDev), stdDev); } public static List intermediateStateDesc() { diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntAggregatorFunctionSupplier.java index 2f43a867bf83e..9475818c918d7 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntAggregatorFunctionSupplier.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntAggregatorFunctionSupplier.java @@ -15,7 +15,10 @@ * This class is generated. Edit {@code AggregatorFunctionSupplierImplementer} instead. */ public final class StdDevIntAggregatorFunctionSupplier implements AggregatorFunctionSupplier { - public StdDevIntAggregatorFunctionSupplier() { + private final boolean stdDev; + + public StdDevIntAggregatorFunctionSupplier(boolean stdDev) { + this.stdDev = stdDev; } @Override @@ -31,13 +34,13 @@ public List groupingIntermediateStateDesc() { @Override public StdDevIntAggregatorFunction aggregator(DriverContext driverContext, List channels) { - return StdDevIntAggregatorFunction.create(driverContext, channels); + return StdDevIntAggregatorFunction.create(driverContext, channels, stdDev); } @Override public StdDevIntGroupingAggregatorFunction groupingAggregator(DriverContext driverContext, List channels) { - return StdDevIntGroupingAggregatorFunction.create(channels, driverContext); + return StdDevIntGroupingAggregatorFunction.create(channels, driverContext, stdDev); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntGroupingAggregatorFunction.java index 858ce287ae6d6..2c0c427b9266e 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevIntGroupingAggregatorFunction.java @@ -38,16 +38,19 @@ public final class StdDevIntGroupingAggregatorFunction implements GroupingAggreg private final DriverContext driverContext; + private final boolean stdDev; + public StdDevIntGroupingAggregatorFunction(List channels, - VarianceStates.GroupingState state, DriverContext driverContext) { + VarianceStates.GroupingState state, DriverContext driverContext, boolean stdDev) { this.channels = channels; this.state = state; this.driverContext = driverContext; + this.stdDev = stdDev; } public static StdDevIntGroupingAggregatorFunction create(List channels, - DriverContext driverContext) { - return new StdDevIntGroupingAggregatorFunction(channels, StdDevIntAggregator.initGrouping(driverContext.bigArrays()), driverContext); + DriverContext driverContext, boolean stdDev) { + return new StdDevIntGroupingAggregatorFunction(channels, StdDevIntAggregator.initGrouping(driverContext.bigArrays(), stdDev), driverContext, stdDev); } public static List intermediateStateDesc() { diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongAggregatorFunction.java index e5563c380bfdd..edf7b4fbecb44 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongAggregatorFunction.java @@ -35,16 +35,19 @@ public final class StdDevLongAggregatorFunction implements AggregatorFunction { private final List channels; + private final boolean stdDev; + public StdDevLongAggregatorFunction(DriverContext driverContext, List channels, - VarianceStates.SingleState state) { + VarianceStates.SingleState state, boolean stdDev) { this.driverContext = driverContext; this.channels = channels; this.state = state; + this.stdDev = stdDev; } public static StdDevLongAggregatorFunction create(DriverContext driverContext, - List channels) { - return new StdDevLongAggregatorFunction(driverContext, channels, StdDevLongAggregator.initSingle()); + List channels, boolean stdDev) { + return new StdDevLongAggregatorFunction(driverContext, channels, StdDevLongAggregator.initSingle(stdDev), stdDev); } public static List intermediateStateDesc() { diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongAggregatorFunctionSupplier.java index 364fc4820c283..bacba99142e41 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongAggregatorFunctionSupplier.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongAggregatorFunctionSupplier.java @@ -15,7 +15,10 @@ * This class is generated. Edit {@code AggregatorFunctionSupplierImplementer} instead. */ public final class StdDevLongAggregatorFunctionSupplier implements AggregatorFunctionSupplier { - public StdDevLongAggregatorFunctionSupplier() { + private final boolean stdDev; + + public StdDevLongAggregatorFunctionSupplier(boolean stdDev) { + this.stdDev = stdDev; } @Override @@ -31,13 +34,13 @@ public List groupingIntermediateStateDesc() { @Override public StdDevLongAggregatorFunction aggregator(DriverContext driverContext, List channels) { - return StdDevLongAggregatorFunction.create(driverContext, channels); + return StdDevLongAggregatorFunction.create(driverContext, channels, stdDev); } @Override public StdDevLongGroupingAggregatorFunction groupingAggregator(DriverContext driverContext, List channels) { - return StdDevLongGroupingAggregatorFunction.create(channels, driverContext); + return StdDevLongGroupingAggregatorFunction.create(channels, driverContext, stdDev); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongGroupingAggregatorFunction.java index 64f0ec2571687..60aea98c728bf 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/StdDevLongGroupingAggregatorFunction.java @@ -37,16 +37,19 @@ public final class StdDevLongGroupingAggregatorFunction implements GroupingAggre private final DriverContext driverContext; + private final boolean stdDev; + public StdDevLongGroupingAggregatorFunction(List channels, - VarianceStates.GroupingState state, DriverContext driverContext) { + VarianceStates.GroupingState state, DriverContext driverContext, boolean stdDev) { this.channels = channels; this.state = state; this.driverContext = driverContext; + this.stdDev = stdDev; } public static StdDevLongGroupingAggregatorFunction create(List channels, - DriverContext driverContext) { - return new StdDevLongGroupingAggregatorFunction(channels, StdDevLongAggregator.initGrouping(driverContext.bigArrays()), driverContext); + DriverContext driverContext, boolean stdDev) { + return new StdDevLongGroupingAggregatorFunction(channels, StdDevLongAggregator.initGrouping(driverContext.bigArrays(), stdDev), driverContext, stdDev); } public static List intermediateStateDesc() { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/VarianceStates.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/VarianceStates.java index 2cf0921205f4f..3a3fb6e7841d4 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/VarianceStates.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/VarianceStates.java @@ -24,8 +24,8 @@ static final class SingleState implements AggregatorState { private final WelfordAlgorithm welfordAlgorithm; - SingleState() { - this(0, 0, 0, true); + SingleState(boolean stdDev) { + this(0, 0, 0, stdDev); } SingleState(double mean, double m2, long count, boolean stdDev) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-StdDevAggregator.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-StdDevAggregator.java.st index 5cb3197ce0e3c..0705a24024875 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-StdDevAggregator.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-StdDevAggregator.java.st @@ -28,35 +28,35 @@ import org.elasticsearch.compute.operator.DriverContext; @GroupingAggregator public class StdDev$Type$Aggregator { - public static StdDevStates.SingleState initSingle() { - return new StdDevStates.SingleState(); + public static VarianceStates.SingleState initSingle(boolean stdDev) { + return new VarianceStates.SingleState(stdDev); } - public static void combine(StdDevStates.SingleState state, $type$ value) { + public static void combine(VarianceStates.SingleState state, $type$ value) { state.add(value); } - public static void combineIntermediate(StdDevStates.SingleState state, double mean, double m2, long count) { + public static void combineIntermediate(VarianceStates.SingleState state, double mean, double m2, long count) { state.combine(mean, m2, count); } - public static Block evaluateFinal(StdDevStates.SingleState state, DriverContext driverContext) { + public static Block evaluateFinal(VarianceStates.SingleState state, DriverContext driverContext) { return state.evaluateFinal(driverContext); } - public static StdDevStates.GroupingState initGrouping(BigArrays bigArrays) { - return new StdDevStates.GroupingState(bigArrays); + public static VarianceStates.GroupingState initGrouping(BigArrays bigArrays, boolean stdDev) { + return new VarianceStates.GroupingState(bigArrays, stdDev); } - public static void combine(StdDevStates.GroupingState current, int groupId, $type$ value) { + public static void combine(VarianceStates.GroupingState current, int groupId, $type$ value) { current.add(groupId, value); } - public static void combineIntermediate(StdDevStates.GroupingState state, int groupId, double mean, double m2, long count) { + public static void combineIntermediate(VarianceStates.GroupingState state, int groupId, double mean, double m2, long count) { state.combine(groupId, mean, m2, count); } - public static Block evaluateFinal(StdDevStates.GroupingState state, IntVector selected, GroupingAggregatorEvaluationContext ctx) { + public static Block evaluateFinal(VarianceStates.GroupingState state, IntVector selected, GroupingAggregatorEvaluationContext ctx) { return state.evaluateFinal(selected, ctx.driverContext()); } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec index d923a5883726c..812ba5616662a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec @@ -596,6 +596,7 @@ max_cost:integer | pod:keyword | time_bucket:datetime | max_cluster_cost:int max_of_stddev_over_time required_capability: ts_command_v0 +required_capability: variance_stddev_over_time // tag::stddev_over_time[] TS k8s | STATS max_stddev_cost=MAX(STDDEV_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) @@ -616,3 +617,51 @@ max_stddev_cost:double | cluster:keyword | time_bucket:datetime 3.835597157621686 | staging | 2024-05-10T00:13:00.000Z 3.8125 | staging | 2024-05-10T00:12:00.000Z ; + +avg_stdvar_over_time +required_capability: ts_command_v0 +required_capability: variance_stddev_over_time +// tag::stdvar_over_time[] +TS k8s +| STATS avg_stdvar_cost=AVG(STDVAR_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) +// end::stdvar_over_time[] +| SORT avg_stdvar_cost DESC, time_bucket DESC, cluster | LIMIT 10; +// tag::stdvar_over_time-result[] +avg_stdvar_cost:double | cluster:keyword | time_bucket:datetime +20.478515625 | staging | 2024-05-10T00:03:00.000Z +16.0 | qa | 2024-05-10T00:21:00.000Z +11.192274305555557 | qa | 2024-05-10T00:18:00.000Z +10.446903935185185 | staging | 2024-05-10T00:09:00.000Z +10.398003472222221 | qa | 2024-05-10T00:17:00.000Z +// end::stdvar_over_time-result[] +10.114583333333332 | qa | 2024-05-10T00:08:00.000Z +7.91015625 | staging | 2024-05-10T00:20:00.000Z +7.355902777777778 | staging | 2024-05-10T00:13:00.000Z +6.5703125 | prod | 2024-05-10T00:02:00.000Z +5.157552083333333 | prod | 2024-05-10T00:18:00.000Z +; + +stddev_sq_and_var_over_time_are_consistent +required_capability: ts_command_v0 +required_capability: variance_stddev_over_time + +TS k8s +| STATS sd=max(stddev_over_time(network.cost)), var=max(stdvar_over_time(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) +| EVAL sd_squared = sd * sd +| KEEP sd, var, sd_squared, cluster, time_bucket +| SORT cluster, time_bucket +| LIMIT 10; + +sd:double | var:double | sd_squared:double | cluster:keyword | time_bucket:datetime +1.4375 | 2.06640625 | 2.06640625 | prod | 2024-05-10T00:00:00.000Z +0.0 | 0.0 | 0.0 | prod | 2024-05-10T00:01:00.000Z +3.625 | 13.140625 | 13.140625 | prod | 2024-05-10T00:02:00.000Z +0.0 | 0.0 | 0.0 | prod | 2024-05-10T00:03:00.000Z +0.0 | 0.0 | 0.0 | prod | 2024-05-10T00:04:00.000Z +2.75 | 7.5625 | 7.5625 | prod | 2024-05-10T00:05:00.000Z +0.0 | 0.0 | 0.0 | prod | 2024-05-10T00:06:00.000Z +1.9375 | 3.75390625 | 3.75390625 | prod | 2024-05-10T00:08:00.000Z +3.406998516126605 | 11.607638888888891 | 11.60763888888889 | prod | 2024-05-10T00:09:00.000Z +0.0 | 0.0 | 0.0 | prod | 2024-05-10T00:10:00.000Z + +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 3acbb4d36899e..e6770e05a39f1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1468,7 +1468,7 @@ public enum Cap { * Percentile over time and other ts-aggregations */ PERCENTILE_OVER_TIME, - + VARIANCE_STDDEV_OVER_TIME, /** * INLINE STATS fix incorrect prunning of null filtering * https://github.com/elastic/elasticsearch/pull/135011 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 17a560cfea7ae..10121f1aa2ce8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -50,6 +50,8 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialExtent; import org.elasticsearch.xpack.esql.expression.function.aggregate.StdDev; import org.elasticsearch.xpack.esql.expression.function.aggregate.StdDevOverTime; +import org.elasticsearch.xpack.esql.expression.function.aggregate.StdVar; +import org.elasticsearch.xpack.esql.expression.function.aggregate.StdVarOverTime; import org.elasticsearch.xpack.esql.expression.function.aggregate.Sum; import org.elasticsearch.xpack.esql.expression.function.aggregate.SumOverTime; import org.elasticsearch.xpack.esql.expression.function.aggregate.Top; @@ -356,6 +358,7 @@ private static FunctionDefinition[][] functions() { def(Percentile.class, bi(Percentile::new), "percentile"), def(Sample.class, bi(Sample::new), "sample"), def(StdDev.class, uni(StdDev::new), "std_dev"), + def(StdVar.class, uni(StdVar::new), "std_var"), def(Sum.class, uni(Sum::new), "sum"), def(Top.class, tri(Top::new), "top"), def(Values.class, uni(Values::new), "values"), @@ -529,6 +532,7 @@ private static FunctionDefinition[][] functions() { def(MinOverTime.class, uni(MinOverTime::new), "min_over_time"), def(SumOverTime.class, uni(SumOverTime::new), "sum_over_time"), def(StdDevOverTime.class, uni(StdDevOverTime::new), "stddev_over_time"), + def(StdVarOverTime.class, uni(StdVarOverTime::new), "stdvar_over_time"), def(CountOverTime.class, uni(CountOverTime::new), "count_over_time"), def(CountDistinctOverTime.class, bi(CountDistinctOverTime::new), "count_distinct_over_time"), def(PresentOverTime.class, uni(PresentOverTime::new), "present_over_time"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateWritables.java index 144d54887b565..00927e71fe9eb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateWritables.java @@ -33,6 +33,7 @@ public static List getNamedWriteables() { SpatialCentroid.ENTRY, SpatialExtent.ENTRY, StdDev.ENTRY, + StdVar.ENTRY, Sum.ENTRY, Top.ENTRY, Values.ENTRY, @@ -40,6 +41,7 @@ public static List getNamedWriteables() { MaxOverTime.ENTRY, AvgOverTime.ENTRY, StdDevOverTime.ENTRY, + StdVarOverTime.ENTRY, Last.ENTRY, LastOverTime.ENTRY, FirstOverTime.ENTRY, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDev.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDev.java index 22a97c2ccb5b3..af4428199e322 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDev.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDev.java @@ -100,13 +100,13 @@ public StdDev withFilter(Expression filter) { public final AggregatorFunctionSupplier supplier() { DataType type = field().dataType(); if (type == DataType.LONG) { - return new StdDevLongAggregatorFunctionSupplier(); + return new StdDevLongAggregatorFunctionSupplier(true); } if (type == DataType.INTEGER) { - return new StdDevIntAggregatorFunctionSupplier(); + return new StdDevIntAggregatorFunctionSupplier(true); } if (type == DataType.DOUBLE) { - return new StdDevDoubleAggregatorFunctionSupplier(); + return new StdDevDoubleAggregatorFunctionSupplier(true); } throw EsqlIllegalArgumentException.illegalDataType(type); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVar.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVar.java new file mode 100644 index 0000000000000..5f606296adfe7 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVar.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.aggregate; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.StdDevDoubleAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.StdDevIntAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.StdDevLongAggregatorFunctionSupplier; +import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.FunctionType; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.planner.ToAggregator; + +import java.io.IOException; +import java.util.List; + +import static java.util.Collections.emptyList; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; + +public class StdVar extends AggregateFunction implements ToAggregator { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "StdVar", StdVar::new); + + @FunctionInfo( + returnType = "double", + description = "The population standard variance of a numeric field.", + type = FunctionType.AGGREGATE + ) + public StdVar(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) { + this(source, field, Literal.TRUE); + } + + public StdVar(Source source, Expression field, Expression filter) { + super(source, field, filter, emptyList()); + } + + private StdVar(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public DataType dataType() { + return DataType.DOUBLE; + } + + @Override + protected Expression.TypeResolution resolveType() { + return isType( + field(), + dt -> dt.isNumeric() && dt != DataType.UNSIGNED_LONG, + sourceText(), + DEFAULT, + "numeric except unsigned_long or counter types" + ); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, StdVar::new, field(), filter()); + } + + @Override + public StdVar replaceChildren(List newChildren) { + return new StdVar(source(), newChildren.get(0), newChildren.get(1)); + } + + public StdVar withFilter(Expression filter) { + return new StdVar(source(), field(), filter); + } + + @Override + public final AggregatorFunctionSupplier supplier() { + DataType type = field().dataType(); + if (type == DataType.LONG) { + return new StdDevLongAggregatorFunctionSupplier(false); + } + if (type == DataType.INTEGER) { + return new StdDevIntAggregatorFunctionSupplier(false); + } + if (type == DataType.DOUBLE) { + return new StdDevDoubleAggregatorFunctionSupplier(false); + } + throw EsqlIllegalArgumentException.illegalDataType(type); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTime.java new file mode 100644 index 0000000000000..b47fefc159683 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTime.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.aggregate; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesTo; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesToLifecycle; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.FunctionType; +import org.elasticsearch.xpack.esql.expression.function.Param; + +import java.io.IOException; +import java.util.List; + +import static java.util.Collections.emptyList; + +/** + * Similar to {@link StdVar}, but it is used to calculate the standard deviation over a time series of values from the given field. + */ +public class StdVarOverTime extends TimeSeriesAggregateFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "StdVarOverTime", + StdVarOverTime::new + ); + + @FunctionInfo( + returnType = "double", + description = "Calculates the population standard deviation over time of a numeric field.", + type = FunctionType.TIME_SERIES_AGGREGATE, + appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0") }, + preview = true, + examples = { @Example(file = "k8s-timeseries", tag = "stdvar_over_time") } + ) + public StdVarOverTime( + Source source, + @Param( + name = "number", + type = { "aggregate_metric_double", "double", "integer", "long" }, + description = "Expression that outputs values to calculate the standard deviation of." + ) Expression field + ) { + this(source, field, Literal.TRUE); + } + + public StdVarOverTime(Source source, Expression field, Expression filter) { + super(source, field, filter, emptyList()); + } + + private StdVarOverTime(StreamInput in) throws IOException { + super(in); + } + + @Override + protected TypeResolution resolveType() { + return perTimeSeriesAggregation().resolveType(); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public DataType dataType() { + return perTimeSeriesAggregation().dataType(); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, StdVarOverTime::new, field(), filter()); + } + + @Override + public StdVarOverTime replaceChildren(List newChildren) { + return new StdVarOverTime(source(), newChildren.get(0), newChildren.get(1)); + } + + @Override + public StdVarOverTime withFilter(Expression filter) { + return new StdVarOverTime(source(), field(), filter); + } + + @Override + public AggregateFunction perTimeSeriesAggregation() { + return new StdVar(source(), field(), filter()); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTimeTests.java new file mode 100644 index 0000000000000..6a935cbd5db13 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTimeTests.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.aggregate; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.List; +import java.util.function.Supplier; + +public class StdVarOverTimeTests extends AbstractFunctionTestCase { + public StdVarOverTimeTests(Supplier testCaseSupplier) { + testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + return StdVarTests.parameters(); + } + + @Override + protected Expression build(Source source, List args) { + return new StdVarOverTime(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java new file mode 100644 index 0000000000000..c3b184b0c9aed --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.aggregate; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.compute.aggregation.WelfordAlgorithm; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase; +import org.elasticsearch.xpack.esql.expression.function.MultiRowTestCaseSupplier; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.hamcrest.Matchers.equalTo; + +public class StdVarTests extends AbstractAggregationTestCase { + public StdVarTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + var suppliers = new ArrayList(); + + Stream.of( + MultiRowTestCaseSupplier.intCases(1, 1000, Integer.MIN_VALUE, Integer.MAX_VALUE, true), + MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true), + MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true) + ).flatMap(List::stream).map(StdVarTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); + + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(suppliers, true); + } + + @Override + protected Expression build(Source source, List args) { + return new StdVar(source, args.get(0)); + } + + private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier) { + return new TestCaseSupplier(List.of(fieldSupplier.type()), () -> { + var fieldTypedData = fieldSupplier.get(); + var fieldValues = fieldTypedData.multiRowData(); + + WelfordAlgorithm welfordAlgorithm = new WelfordAlgorithm(false); + + for (var fieldValue : fieldValues) { + var value = ((Number) fieldValue).doubleValue(); + welfordAlgorithm.add(value); + } + var result = welfordAlgorithm.evaluate(); + var expected = Double.isFinite(result) ? result : null; + return new TestCaseSupplier.TestCase( + List.of(fieldTypedData), + standardAggregatorName("StdVar", fieldSupplier.type()), + DataType.DOUBLE, + equalTo(expected) + ); + }); + } +} From 9dde5826ebd73dd504387c188cb0e3ce3a9de9d4 Mon Sep 17 00:00:00 2001 From: Pablo Date: Thu, 16 Oct 2025 16:58:16 -0700 Subject: [PATCH 04/24] fixup --- .../org/elasticsearch/compute/aggregation/WelfordAlgorithm.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/WelfordAlgorithm.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/WelfordAlgorithm.java index d1dce9e93aaf6..8168ed2ee7962 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/WelfordAlgorithm.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/WelfordAlgorithm.java @@ -81,7 +81,7 @@ public void add(double meanValue, double m2Value, long countValue) { } public double evaluate() { - if (!stdDev) { + if (stdDev == false) { return count < 2 ? 0 : m2 / count; } else { return count < 2 ? 0 : Math.sqrt(m2 / count); From 580929d5abd7667fb5665930f3fc607a233ccf18 Mon Sep 17 00:00:00 2001 From: Pablo Date: Thu, 16 Oct 2025 19:41:31 -0700 Subject: [PATCH 05/24] fixup --- .../functions/description/std_var.md | 6 +++ .../_snippets/functions/examples/std_var.md | 14 ++++++ .../_snippets/functions/layout/std_var.md | 23 +++++++++ .../_snippets/functions/parameters/std_var.md | 7 +++ .../esql/_snippets/functions/types/std_var.md | 10 ++++ .../esql/images/functions/std_var.svg | 1 + .../kibana/definition/functions/std_var.json | 49 +++++++++++++++++++ .../esql/kibana/docs/functions/std_var.md | 9 ++++ .../src/main/resources/stats.csv-spec | 14 ++++++ .../expression/function/aggregate/StdVar.java | 6 ++- .../function/aggregate/StdVarTests.java | 3 +- 11 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 docs/reference/query-languages/esql/_snippets/functions/description/std_var.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/examples/std_var.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/layout/std_var.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/parameters/std_var.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/types/std_var.md create mode 100644 docs/reference/query-languages/esql/images/functions/std_var.svg create mode 100644 docs/reference/query-languages/esql/kibana/definition/functions/std_var.json create mode 100644 docs/reference/query-languages/esql/kibana/docs/functions/std_var.md diff --git a/docs/reference/query-languages/esql/_snippets/functions/description/std_var.md b/docs/reference/query-languages/esql/_snippets/functions/description/std_var.md new file mode 100644 index 0000000000000..0772559a8a5d9 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/description/std_var.md @@ -0,0 +1,6 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Description** + +The population standard variance of a numeric field. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/std_var.md b/docs/reference/query-languages/esql/_snippets/functions/examples/std_var.md new file mode 100644 index 0000000000000..d10976b1794d3 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/std_var.md @@ -0,0 +1,14 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Example** + +```esql +FROM employees +| STATS std_dev_height = STD_DEV(height) +``` + +| std_dev_height:double | +| --- | +| 0.2063704 | + + diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/std_var.md b/docs/reference/query-languages/esql/_snippets/functions/layout/std_var.md new file mode 100644 index 0000000000000..d19ab9ce11eb6 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/layout/std_var.md @@ -0,0 +1,23 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +## `STD_VAR` [esql-std_var] + +**Syntax** + +:::{image} ../../../images/functions/std_var.svg +:alt: Embedded +:class: text-center +::: + + +:::{include} ../parameters/std_var.md +::: + +:::{include} ../description/std_var.md +::: + +:::{include} ../types/std_var.md +::: + +:::{include} ../examples/std_var.md +::: diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/std_var.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/std_var.md new file mode 100644 index 0000000000000..e003b3562deeb --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/parameters/std_var.md @@ -0,0 +1,7 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Parameters** + +`number` +: + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/std_var.md b/docs/reference/query-languages/esql/_snippets/functions/types/std_var.md new file mode 100644 index 0000000000000..e204309f17bb1 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/std_var.md @@ -0,0 +1,10 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| number | result | +| --- | --- | +| double | double | +| integer | double | +| long | double | + diff --git a/docs/reference/query-languages/esql/images/functions/std_var.svg b/docs/reference/query-languages/esql/images/functions/std_var.svg new file mode 100644 index 0000000000000..53f007458df97 --- /dev/null +++ b/docs/reference/query-languages/esql/images/functions/std_var.svg @@ -0,0 +1 @@ +STD_VAR(number) \ No newline at end of file diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/std_var.json b/docs/reference/query-languages/esql/kibana/definition/functions/std_var.json new file mode 100644 index 0000000000000..9da94c0adde88 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/functions/std_var.json @@ -0,0 +1,49 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "agg", + "name" : "std_var", + "description" : "The population standard variance of a numeric field.", + "signatures" : [ + { + "params" : [ + { + "name" : "number", + "type" : "double", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "integer", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "long", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "FROM employees\n| STATS std_dev_height = STD_DEV(height)" + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/std_var.md b/docs/reference/query-languages/esql/kibana/docs/functions/std_var.md new file mode 100644 index 0000000000000..8ab27cc6c1d1a --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/docs/functions/std_var.md @@ -0,0 +1,9 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +### STD VAR +The population standard variance of a numeric field. + +```esql +FROM employees +| STATS std_dev_height = STD_DEV(height) +``` diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index 61ca316a39dbc..a9637662755c0 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -2969,6 +2969,20 @@ std_dev_height:double // end::stdev-result[] ; +stdVariance + +// tag::stdVar[] +FROM employees +| STATS std_var_height = STD_VAR(height) +// end::stdVar[] +| EVAL std_var_height = ROUND(std_var_height, 7) +; + +// tag::stdVar-result[] +std_var_height:double +0.0426061 +// end::stdVar-result[] + stdDeviationNested required_capability: std_dev // tag::docsStatsStdDevNestedExpression[] diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVar.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVar.java index 5f606296adfe7..19eec727fa714 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVar.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVar.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.FunctionType; import org.elasticsearch.xpack.esql.expression.function.Param; @@ -37,7 +38,10 @@ public class StdVar extends AggregateFunction implements ToAggregator { @FunctionInfo( returnType = "double", description = "The population standard variance of a numeric field.", - type = FunctionType.AGGREGATE + type = FunctionType.AGGREGATE, + examples = { + @Example(file = "stats", tag = "stdev") + } ) public StdVar(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) { this(source, field, Literal.TRUE); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java index c3b184b0c9aed..c7d5cca02e812 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java @@ -64,7 +64,8 @@ private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier var expected = Double.isFinite(result) ? result : null; return new TestCaseSupplier.TestCase( List.of(fieldTypedData), - standardAggregatorName("StdVar", fieldSupplier.type()), + // Note that this stddev because it's the operator that implements both of these + standardAggregatorName("StdDev", fieldSupplier.type()), DataType.DOUBLE, equalTo(expected) ); From 95c1df8d144a1d7756245df00979cc017d119688 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 17 Oct 2025 02:47:20 +0000 Subject: [PATCH 06/24] [CI] Auto commit changes from spotless --- .../xpack/esql/expression/function/aggregate/StdVar.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVar.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVar.java index 19eec727fa714..a02bc9c60e53e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVar.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVar.java @@ -39,9 +39,7 @@ public class StdVar extends AggregateFunction implements ToAggregator { returnType = "double", description = "The population standard variance of a numeric field.", type = FunctionType.AGGREGATE, - examples = { - @Example(file = "stats", tag = "stdev") - } + examples = { @Example(file = "stats", tag = "stdev") } ) public StdVar(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) { this(source, field, Literal.TRUE); From d302fa78f41638975ac4f26f45149d0264a3a05c Mon Sep 17 00:00:00 2001 From: Pablo Date: Thu, 16 Oct 2025 19:51:38 -0700 Subject: [PATCH 07/24] fixup --- .../esql/expression/function/aggregate/StdDevOverTime.java | 2 +- .../esql/expression/function/aggregate/StdVarOverTime.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTime.java index 7b0b0d8f65e8d..fbc9761c4b307 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTime.java @@ -40,7 +40,7 @@ public class StdDevOverTime extends TimeSeriesAggregateFunction { returnType = "double", description = "Calculates the population standard deviation over time of a numeric field.", type = FunctionType.TIME_SERIES_AGGREGATE, - appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0") }, + appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.3.0") }, preview = true, examples = { @Example(file = "k8s-timeseries", tag = "stddev_over_time") } ) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTime.java index b47fefc159683..b304f3fd7560a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTime.java @@ -40,7 +40,7 @@ public class StdVarOverTime extends TimeSeriesAggregateFunction { returnType = "double", description = "Calculates the population standard deviation over time of a numeric field.", type = FunctionType.TIME_SERIES_AGGREGATE, - appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0") }, + appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.3.0") }, preview = true, examples = { @Example(file = "k8s-timeseries", tag = "stdvar_over_time") } ) From e6ed713a5adfacf8333b6290e269a2bbae692770 Mon Sep 17 00:00:00 2001 From: Pablo Date: Thu, 16 Oct 2025 20:45:53 -0700 Subject: [PATCH 08/24] fixup --- .../esql/qa/testFixtures/src/main/resources/stats.csv-spec | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index a9637662755c0..e29b8ef4071e8 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -2982,6 +2982,7 @@ FROM employees std_var_height:double 0.0426061 // end::stdVar-result[] +; stdDeviationNested required_capability: std_dev From 890624b10251dbf4f5ee99acb09c97c716889d98 Mon Sep 17 00:00:00 2001 From: Pablo Date: Fri, 17 Oct 2025 09:21:25 -0700 Subject: [PATCH 09/24] fixup --- .../compute/aggregation/WelfordAlgorithm.java | 15 +++++++++++---- .../src/main/resources/stats.csv-spec | 6 +++--- .../function/aggregate/StdDevTests.java | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/WelfordAlgorithm.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/WelfordAlgorithm.java index 8168ed2ee7962..79cfa3e43c63a 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/WelfordAlgorithm.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/WelfordAlgorithm.java @@ -34,14 +34,21 @@ public long count() { return count; } - public WelfordAlgorithm() { - this(0, 0, 0, true); - } - + /** + * Creates a new WelfordAlgorithm instance. + * @param stdDev if true, compute standard deviation, otherwise compute variance + */ public WelfordAlgorithm(boolean stdDev) { this(0, 0, 0, stdDev); } + /** + * Creates a new WelfordAlgorithm instance with the given state. + * @param mean the mean calculated so far + * @param m2 the sum of squares of differences from the current mean + * @param count the number of values added so far + * @param stdDev if true, compute standard deviation, otherwise compute variance + */ public WelfordAlgorithm(double mean, double m2, long count, boolean stdDev) { this.mean = mean; this.m2 = m2; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index e29b8ef4071e8..3b724d261624a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -2971,16 +2971,16 @@ std_dev_height:double stdVariance -// tag::stdVar[] +// tag::stdvar[] FROM employees | STATS std_var_height = STD_VAR(height) // end::stdVar[] | EVAL std_var_height = ROUND(std_var_height, 7) ; -// tag::stdVar-result[] +// tag::stdvar-result[] std_var_height:double -0.0426061 +0.0425888 // end::stdVar-result[] ; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevTests.java index f78e04ec94e82..0d0997d95643c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevTests.java @@ -54,7 +54,7 @@ private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier var fieldTypedData = fieldSupplier.get(); var fieldValues = fieldTypedData.multiRowData(); - WelfordAlgorithm welfordAlgorithm = new WelfordAlgorithm(); + WelfordAlgorithm welfordAlgorithm = new WelfordAlgorithm(true); for (var fieldValue : fieldValues) { var value = ((Number) fieldValue).doubleValue(); From a071eea747dcc7c874654225b5376b6df91e7440 Mon Sep 17 00:00:00 2001 From: Pablo Date: Fri, 17 Oct 2025 09:25:48 -0700 Subject: [PATCH 10/24] fixup --- .../compute/aggregation/VarianceStates.java | 12 +++++---- .../compute/aggregation/WelfordAlgorithm.java | 27 +++++++------------ 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/VarianceStates.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/VarianceStates.java index 3a3fb6e7841d4..a4b6b2a04d593 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/VarianceStates.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/VarianceStates.java @@ -23,13 +23,15 @@ private VarianceStates() {} static final class SingleState implements AggregatorState { private final WelfordAlgorithm welfordAlgorithm; + private final boolean stdDev; SingleState(boolean stdDev) { this(0, 0, 0, stdDev); } SingleState(double mean, double m2, long count, boolean stdDev) { - this.welfordAlgorithm = new WelfordAlgorithm(mean, m2, count, stdDev); + this.welfordAlgorithm = new WelfordAlgorithm(mean, m2, count); + this.stdDev = stdDev; } public void add(long value) { @@ -73,7 +75,7 @@ public long count() { } public double evaluateFinal() { - return welfordAlgorithm.evaluate(); + return welfordAlgorithm.evaluate(stdDev); } public Block evaluateFinal(DriverContext driverContext) { @@ -117,7 +119,7 @@ public void combine(int groupId, double meanValue, double m2Value, long countVal ensureCapacity(groupId); var state = states.get(groupId); if (state == null) { - state = new WelfordAlgorithm(meanValue, m2Value, countValue, stdDev); + state = new WelfordAlgorithm(meanValue, m2Value, countValue); states.set(groupId, state); } else { state.add(meanValue, m2Value, countValue); @@ -128,7 +130,7 @@ public WelfordAlgorithm getOrSet(int groupId) { ensureCapacity(groupId); var state = states.get(groupId); if (state == null) { - state = new WelfordAlgorithm(stdDev); + state = new WelfordAlgorithm(); states.set(groupId, state); } return state; @@ -191,7 +193,7 @@ public Block evaluateFinal(IntVector selected, DriverContext driverContext) { if (count == 0 || Double.isFinite(m2) == false) { builder.appendNull(); } else { - builder.appendDouble(st.evaluate()); + builder.appendDouble(st.evaluate(stdDev)); } } else { builder.appendNull(); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/WelfordAlgorithm.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/WelfordAlgorithm.java index 79cfa3e43c63a..2ab4c75624cb2 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/WelfordAlgorithm.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/WelfordAlgorithm.java @@ -19,8 +19,6 @@ public final class WelfordAlgorithm { private double mean; private double m2; private long count; - // If true, compute standard deviation, otherwise compute variance - private final boolean stdDev; public double mean() { return mean; @@ -34,26 +32,14 @@ public long count() { return count; } - /** - * Creates a new WelfordAlgorithm instance. - * @param stdDev if true, compute standard deviation, otherwise compute variance - */ - public WelfordAlgorithm(boolean stdDev) { - this(0, 0, 0, stdDev); + public WelfordAlgorithm() { + this(0, 0, 0); } - /** - * Creates a new WelfordAlgorithm instance with the given state. - * @param mean the mean calculated so far - * @param m2 the sum of squares of differences from the current mean - * @param count the number of values added so far - * @param stdDev if true, compute standard deviation, otherwise compute variance - */ - public WelfordAlgorithm(double mean, double m2, long count, boolean stdDev) { + public WelfordAlgorithm(double mean, double m2, long count) { this.mean = mean; this.m2 = m2; this.count = count; - this.stdDev = stdDev; } public void add(int value) { @@ -87,7 +73,12 @@ public void add(double meanValue, double m2Value, long countValue) { count += countValue; } - public double evaluate() { + /** + * Evaluate the variance or standard deviation. + * @param stdDev if true, compute standard deviation, otherwise variance + * @return + */ + public double evaluate(boolean stdDev) { if (stdDev == false) { return count < 2 ? 0 : m2 / count; } else { From 319054d296354f052b914e196a9d5e79f9578357 Mon Sep 17 00:00:00 2001 From: Pablo Date: Fri, 17 Oct 2025 09:32:57 -0700 Subject: [PATCH 11/24] more fixup --- .../xpack/esql/expression/function/aggregate/StdDevTests.java | 4 ++-- .../xpack/esql/expression/function/aggregate/StdVarTests.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevTests.java index 0d0997d95643c..f070852dafddb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevTests.java @@ -54,13 +54,13 @@ private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier var fieldTypedData = fieldSupplier.get(); var fieldValues = fieldTypedData.multiRowData(); - WelfordAlgorithm welfordAlgorithm = new WelfordAlgorithm(true); + WelfordAlgorithm welfordAlgorithm = new WelfordAlgorithm(); for (var fieldValue : fieldValues) { var value = ((Number) fieldValue).doubleValue(); welfordAlgorithm.add(value); } - var result = welfordAlgorithm.evaluate(); + var result = welfordAlgorithm.evaluate(true); var expected = Double.isFinite(result) ? result : null; return new TestCaseSupplier.TestCase( List.of(fieldTypedData), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java index c7d5cca02e812..6dc0a39c2689a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java @@ -54,13 +54,13 @@ private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier var fieldTypedData = fieldSupplier.get(); var fieldValues = fieldTypedData.multiRowData(); - WelfordAlgorithm welfordAlgorithm = new WelfordAlgorithm(false); + WelfordAlgorithm welfordAlgorithm = new WelfordAlgorithm(); for (var fieldValue : fieldValues) { var value = ((Number) fieldValue).doubleValue(); welfordAlgorithm.add(value); } - var result = welfordAlgorithm.evaluate(); + var result = welfordAlgorithm.evaluate(false); var expected = Double.isFinite(result) ? result : null; return new TestCaseSupplier.TestCase( List.of(fieldTypedData), From 23557d880f7af971100ca4642e9c7a5062da6652 Mon Sep 17 00:00:00 2001 From: Pablo Date: Fri, 17 Oct 2025 09:47:46 -0700 Subject: [PATCH 12/24] fixup - generated docs --- .../functions/description/stddev_over_time.md | 6 +++ .../functions/description/stdvar_over_time.md | 6 +++ .../functions/examples/stddev_over_time.md | 19 +++++++ .../functions/examples/stdvar_over_time.md | 18 +++++++ .../functions/layout/stddev_over_time.md | 27 ++++++++++ .../functions/layout/stdvar_over_time.md | 27 ++++++++++ .../functions/parameters/stddev_over_time.md | 7 +++ .../functions/parameters/stdvar_over_time.md | 7 +++ .../functions/types/stddev_over_time.md | 10 ++++ .../functions/types/stdvar_over_time.md | 10 ++++ .../images/functions/stddev_over_time.svg | 1 + .../images/functions/stdvar_over_time.svg | 1 + .../functions/stddev_over_time.json | 49 +++++++++++++++++++ .../functions/stdvar_over_time.json | 49 +++++++++++++++++++ .../kibana/docs/functions/stddev_over_time.md | 9 ++++ .../kibana/docs/functions/stdvar_over_time.md | 9 ++++ .../function/aggregate/StdDevOverTime.java | 2 +- .../function/aggregate/StdVarOverTime.java | 6 +-- ...imeTests.java => StddevOverTimeTests.java} | 4 +- ...imeTests.java => StdvarOverTimeTests.java} | 4 +- 20 files changed, 263 insertions(+), 8 deletions(-) create mode 100644 docs/reference/query-languages/esql/_snippets/functions/description/stddev_over_time.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/description/stdvar_over_time.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/examples/stddev_over_time.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/examples/stdvar_over_time.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/layout/stddev_over_time.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/layout/stdvar_over_time.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/parameters/stddev_over_time.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/parameters/stdvar_over_time.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/types/stddev_over_time.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/types/stdvar_over_time.md create mode 100644 docs/reference/query-languages/esql/images/functions/stddev_over_time.svg create mode 100644 docs/reference/query-languages/esql/images/functions/stdvar_over_time.svg create mode 100644 docs/reference/query-languages/esql/kibana/definition/functions/stddev_over_time.json create mode 100644 docs/reference/query-languages/esql/kibana/definition/functions/stdvar_over_time.json create mode 100644 docs/reference/query-languages/esql/kibana/docs/functions/stddev_over_time.md create mode 100644 docs/reference/query-languages/esql/kibana/docs/functions/stdvar_over_time.md rename x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/{StdDevOverTimeTests.java => StddevOverTimeTests.java} (89%) rename x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/{StdVarOverTimeTests.java => StdvarOverTimeTests.java} (89%) diff --git a/docs/reference/query-languages/esql/_snippets/functions/description/stddev_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/description/stddev_over_time.md new file mode 100644 index 0000000000000..42db5b4412aa9 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/description/stddev_over_time.md @@ -0,0 +1,6 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Description** + +Calculates the population standard deviation over time of a numeric field. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/description/stdvar_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/description/stdvar_over_time.md new file mode 100644 index 0000000000000..67b3a64b7bceb --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/description/stdvar_over_time.md @@ -0,0 +1,6 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Description** + +Calculates the population standard variance over time of a numeric field. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/stddev_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/examples/stddev_over_time.md new file mode 100644 index 0000000000000..20780fdca810b --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/stddev_over_time.md @@ -0,0 +1,19 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Example** + +```esql +TS k8s +| STATS max_stddev_cost=MAX(STDDEV_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) +``` + +| max_stddev_cost:double | cluster:keyword | time_bucket:datetime | +| --- | --- | --- | +| 5.4375 | staging | 2024-05-10T00:03:00.000Z | +| 5.1875 | staging | 2024-05-10T00:09:00.000Z | +| 4.097763617714749 | qa | 2024-05-10T00:18:00.000Z | +| 4.0 | qa | 2024-05-10T00:21:00.000Z | +| 3.9375 | staging | 2024-05-10T00:20:00.000Z | +| 3.9335297443898907 | prod | 2024-05-10T00:18:00.000Z | + + diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/stdvar_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/examples/stdvar_over_time.md new file mode 100644 index 0000000000000..8639e7b16d9a2 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/stdvar_over_time.md @@ -0,0 +1,18 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Example** + +```esql +TS k8s +| STATS avg_stdvar_cost=AVG(STDVAR_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) +``` + +| avg_stdvar_cost:double | cluster:keyword | time_bucket:datetime | +| --- | --- | --- | +| 20.478515625 | staging | 2024-05-10T00:03:00.000Z | +| 16.0 | qa | 2024-05-10T00:21:00.000Z | +| 11.192274305555557 | qa | 2024-05-10T00:18:00.000Z | +| 10.446903935185185 | staging | 2024-05-10T00:09:00.000Z | +| 10.398003472222221 | qa | 2024-05-10T00:17:00.000Z | + + diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/stddev_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/layout/stddev_over_time.md new file mode 100644 index 0000000000000..e991631595071 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/layout/stddev_over_time.md @@ -0,0 +1,27 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +## `STDDEV_OVER_TIME` [esql-stddev_over_time] +```{applies_to} +stack: preview 9.3.0 +serverless: preview +``` + +**Syntax** + +:::{image} ../../../images/functions/stddev_over_time.svg +:alt: Embedded +:class: text-center +::: + + +:::{include} ../parameters/stddev_over_time.md +::: + +:::{include} ../description/stddev_over_time.md +::: + +:::{include} ../types/stddev_over_time.md +::: + +:::{include} ../examples/stddev_over_time.md +::: diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/stdvar_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/layout/stdvar_over_time.md new file mode 100644 index 0000000000000..29c6e73c1f785 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/layout/stdvar_over_time.md @@ -0,0 +1,27 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +## `STDVAR_OVER_TIME` [esql-stdvar_over_time] +```{applies_to} +stack: preview 9.3.0 +serverless: preview +``` + +**Syntax** + +:::{image} ../../../images/functions/stdvar_over_time.svg +:alt: Embedded +:class: text-center +::: + + +:::{include} ../parameters/stdvar_over_time.md +::: + +:::{include} ../description/stdvar_over_time.md +::: + +:::{include} ../types/stdvar_over_time.md +::: + +:::{include} ../examples/stdvar_over_time.md +::: diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/stddev_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/stddev_over_time.md new file mode 100644 index 0000000000000..0ac6e4bf13042 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/parameters/stddev_over_time.md @@ -0,0 +1,7 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Parameters** + +`number` +: Expression that outputs values to calculate the standard deviation of. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/stdvar_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/stdvar_over_time.md new file mode 100644 index 0000000000000..52a0a163e5a9d --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/parameters/stdvar_over_time.md @@ -0,0 +1,7 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Parameters** + +`number` +: Expression that outputs values to calculate the variance of. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/stddev_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/types/stddev_over_time.md new file mode 100644 index 0000000000000..e204309f17bb1 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/stddev_over_time.md @@ -0,0 +1,10 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| number | result | +| --- | --- | +| double | double | +| integer | double | +| long | double | + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/stdvar_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/types/stdvar_over_time.md new file mode 100644 index 0000000000000..e204309f17bb1 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/stdvar_over_time.md @@ -0,0 +1,10 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| number | result | +| --- | --- | +| double | double | +| integer | double | +| long | double | + diff --git a/docs/reference/query-languages/esql/images/functions/stddev_over_time.svg b/docs/reference/query-languages/esql/images/functions/stddev_over_time.svg new file mode 100644 index 0000000000000..c314d6a193df8 --- /dev/null +++ b/docs/reference/query-languages/esql/images/functions/stddev_over_time.svg @@ -0,0 +1 @@ +STDDEV_OVER_TIME(number) \ No newline at end of file diff --git a/docs/reference/query-languages/esql/images/functions/stdvar_over_time.svg b/docs/reference/query-languages/esql/images/functions/stdvar_over_time.svg new file mode 100644 index 0000000000000..0c0e256a06780 --- /dev/null +++ b/docs/reference/query-languages/esql/images/functions/stdvar_over_time.svg @@ -0,0 +1 @@ +STDVAR_OVER_TIME(number) \ No newline at end of file diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/stddev_over_time.json b/docs/reference/query-languages/esql/kibana/definition/functions/stddev_over_time.json new file mode 100644 index 0000000000000..2a83817caf616 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/functions/stddev_over_time.json @@ -0,0 +1,49 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "time_series_agg", + "name" : "stddev_over_time", + "description" : "Calculates the population standard deviation over time of a numeric field.", + "signatures" : [ + { + "params" : [ + { + "name" : "number", + "type" : "double", + "optional" : false, + "description" : "Expression that outputs values to calculate the standard deviation of." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "integer", + "optional" : false, + "description" : "Expression that outputs values to calculate the standard deviation of." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "long", + "optional" : false, + "description" : "Expression that outputs values to calculate the standard deviation of." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "TS k8s\n| STATS max_stddev_cost=MAX(STDDEV_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute)" + ], + "preview" : true, + "snapshot_only" : false +} diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/stdvar_over_time.json b/docs/reference/query-languages/esql/kibana/definition/functions/stdvar_over_time.json new file mode 100644 index 0000000000000..41f1d968e1e74 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/functions/stdvar_over_time.json @@ -0,0 +1,49 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "time_series_agg", + "name" : "stdvar_over_time", + "description" : "Calculates the population standard variance over time of a numeric field.", + "signatures" : [ + { + "params" : [ + { + "name" : "number", + "type" : "double", + "optional" : false, + "description" : "Expression that outputs values to calculate the variance of." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "integer", + "optional" : false, + "description" : "Expression that outputs values to calculate the variance of." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "long", + "optional" : false, + "description" : "Expression that outputs values to calculate the variance of." + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + "TS k8s\n| STATS avg_stdvar_cost=AVG(STDVAR_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute)" + ], + "preview" : true, + "snapshot_only" : false +} diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/stddev_over_time.md b/docs/reference/query-languages/esql/kibana/docs/functions/stddev_over_time.md new file mode 100644 index 0000000000000..decd95b2fefdc --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/docs/functions/stddev_over_time.md @@ -0,0 +1,9 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +### STDDEV OVER TIME +Calculates the population standard deviation over time of a numeric field. + +```esql +TS k8s +| STATS max_stddev_cost=MAX(STDDEV_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) +``` diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/stdvar_over_time.md b/docs/reference/query-languages/esql/kibana/docs/functions/stdvar_over_time.md new file mode 100644 index 0000000000000..46ea25b048b70 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/docs/functions/stdvar_over_time.md @@ -0,0 +1,9 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +### STDVAR OVER TIME +Calculates the population standard variance over time of a numeric field. + +```esql +TS k8s +| STATS avg_stdvar_cost=AVG(STDVAR_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) +``` diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTime.java index fbc9761c4b307..7f902e0c1acd7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTime.java @@ -48,7 +48,7 @@ public StdDevOverTime( Source source, @Param( name = "number", - type = { "aggregate_metric_double", "double", "integer", "long" }, + type = { "double", "integer", "long" }, description = "Expression that outputs values to calculate the standard deviation of." ) Expression field ) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTime.java index b304f3fd7560a..a7cb78f330e3c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTime.java @@ -38,7 +38,7 @@ public class StdVarOverTime extends TimeSeriesAggregateFunction { @FunctionInfo( returnType = "double", - description = "Calculates the population standard deviation over time of a numeric field.", + description = "Calculates the population standard variance over time of a numeric field.", type = FunctionType.TIME_SERIES_AGGREGATE, appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.3.0") }, preview = true, @@ -48,8 +48,8 @@ public StdVarOverTime( Source source, @Param( name = "number", - type = { "aggregate_metric_double", "double", "integer", "long" }, - description = "Expression that outputs values to calculate the standard deviation of." + type = { "double", "integer", "long" }, + description = "Expression that outputs values to calculate the variance of." ) Expression field ) { this(source, field, Literal.TRUE); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StddevOverTimeTests.java similarity index 89% rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTimeTests.java rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StddevOverTimeTests.java index cf537ddb6c27d..8956ec1ac13a3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTimeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StddevOverTimeTests.java @@ -17,8 +17,8 @@ import java.util.List; import java.util.function.Supplier; -public class StdDevOverTimeTests extends AbstractFunctionTestCase { - public StdDevOverTimeTests(Supplier testCaseSupplier) { +public class StddevOverTimeTests extends AbstractFunctionTestCase { + public StddevOverTimeTests(Supplier testCaseSupplier) { testCase = testCaseSupplier.get(); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdvarOverTimeTests.java similarity index 89% rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTimeTests.java rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdvarOverTimeTests.java index 6a935cbd5db13..1ee1b478cce05 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTimeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdvarOverTimeTests.java @@ -17,8 +17,8 @@ import java.util.List; import java.util.function.Supplier; -public class StdVarOverTimeTests extends AbstractFunctionTestCase { - public StdVarOverTimeTests(Supplier testCaseSupplier) { +public class StdvarOverTimeTests extends AbstractFunctionTestCase { + public StdvarOverTimeTests(Supplier testCaseSupplier) { testCase = testCaseSupplier.get(); } From 8a35d29323969b90b1d6beebe36843385766314c Mon Sep 17 00:00:00 2001 From: Pablo Date: Fri, 17 Oct 2025 10:06:27 -0700 Subject: [PATCH 13/24] foixup --- .../esql/qa/testFixtures/src/main/resources/stats.csv-spec | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index 3b724d261624a..fee3fa3f61624 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -2970,6 +2970,7 @@ std_dev_height:double ; stdVariance +required_capability: VARIANCE_STDDEV_OVER_TIME // tag::stdvar[] FROM employees From c3e890a18cf939815211d6a403bd382dc2f65e89 Mon Sep 17 00:00:00 2001 From: Pablo Date: Fri, 17 Oct 2025 11:06:33 -0700 Subject: [PATCH 14/24] fix test --- .../esql/qa/testFixtures/src/main/resources/stats.csv-spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index fee3fa3f61624..023bfd0076d78 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -2970,7 +2970,7 @@ std_dev_height:double ; stdVariance -required_capability: VARIANCE_STDDEV_OVER_TIME +required_capability: variance_stddev_over_time // tag::stdvar[] FROM employees From 81127722bc8cbfdc3ac3a5be8c9f7c6071edb8b5 Mon Sep 17 00:00:00 2001 From: Pablo Date: Fri, 17 Oct 2025 12:09:05 -0700 Subject: [PATCH 15/24] fixup --- .../xpack/esql/expression/function/aggregate/StdVarTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java index 6dc0a39c2689a..300705bbb3eec 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java @@ -41,7 +41,7 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true) ).flatMap(List::stream).map(StdVarTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); - return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(suppliers, true); + return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers, true); } @Override From 947e7b4190788a6c10a9eebb96ebc24780d44611 Mon Sep 17 00:00:00 2001 From: Pablo Date: Fri, 17 Oct 2025 15:38:20 -0700 Subject: [PATCH 16/24] comments --- .../main/resources/k8s-timeseries.csv-spec | 74 ++++++++++--------- .../src/main/resources/stats.csv-spec | 38 +++++++++- 2 files changed, 76 insertions(+), 36 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec index 812ba5616662a..6ab48109b533d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec @@ -601,21 +601,23 @@ required_capability: variance_stddev_over_time TS k8s | STATS max_stddev_cost=MAX(STDDEV_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) // end::stddev_over_time[] +| EVAL max_stddev_cost = ROUND(max_stddev_cost, 6) | SORT max_stddev_cost DESC, time_bucket DESC, cluster | LIMIT 10; // tag::stddev_over_time-result[] -max_stddev_cost:double | cluster:keyword | time_bucket:datetime -5.4375 | staging | 2024-05-10T00:03:00.000Z -5.1875 | staging | 2024-05-10T00:09:00.000Z -4.097763617714749 | qa | 2024-05-10T00:18:00.000Z -4.0 | qa | 2024-05-10T00:21:00.000Z -3.9375 | staging | 2024-05-10T00:20:00.000Z -3.9335297443898907 | prod | 2024-05-10T00:18:00.000Z +cluster:keyword | time_bucket:datetime | max_stddev_cost:double +staging | 2024-05-10T00:03:00.000Z | 5.4375 +staging | 2024-05-10T00:09:00.000Z | 5.1875 +qa | 2024-05-10T00:18:00.000Z | 4.097764 +qa | 2024-05-10T00:21:00.000Z | 4.0 // end::stddev_over_time-result[] -3.8944404818493075 | qa | 2024-05-10T00:08:00.000Z -3.8823854350987186 | qa | 2024-05-10T00:17:00.000Z -3.835597157621686 | staging | 2024-05-10T00:13:00.000Z -3.8125 | staging | 2024-05-10T00:12:00.000Z +staging | 2024-05-10T00:20:00.000Z | 3.9375 +prod | 2024-05-10T00:18:00.000Z | 3.93353 +qa | 2024-05-10T00:08:00.000Z | 3.89444 +qa | 2024-05-10T00:17:00.000Z | 3.882385 +staging | 2024-05-10T00:13:00.000Z | 3.835597 +staging | 2024-05-10T00:12:00.000Z | 3.8125 + ; avg_stdvar_over_time @@ -625,20 +627,21 @@ required_capability: variance_stddev_over_time TS k8s | STATS avg_stdvar_cost=AVG(STDVAR_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) // end::stdvar_over_time[] +| EVAL avg_stdvar_cost = ROUND(avg_stdvar_cost, 6) | SORT avg_stdvar_cost DESC, time_bucket DESC, cluster | LIMIT 10; // tag::stdvar_over_time-result[] -avg_stdvar_cost:double | cluster:keyword | time_bucket:datetime -20.478515625 | staging | 2024-05-10T00:03:00.000Z -16.0 | qa | 2024-05-10T00:21:00.000Z -11.192274305555557 | qa | 2024-05-10T00:18:00.000Z -10.446903935185185 | staging | 2024-05-10T00:09:00.000Z -10.398003472222221 | qa | 2024-05-10T00:17:00.000Z -// end::stdvar_over_time-result[] -10.114583333333332 | qa | 2024-05-10T00:08:00.000Z -7.91015625 | staging | 2024-05-10T00:20:00.000Z -7.355902777777778 | staging | 2024-05-10T00:13:00.000Z -6.5703125 | prod | 2024-05-10T00:02:00.000Z -5.157552083333333 | prod | 2024-05-10T00:18:00.000Z +cluster:keyword | time_bucket:datetime | avg_stdvar_cost:double +staging | 2024-05-10T00:03:00.000Z | 20.478516 +qa | 2024-05-10T00:21:00.000Z | 16.0 +qa | 2024-05-10T00:18:00.000Z | 11.192274 +staging | 2024-05-10T00:09:00.000Z | 10.446904 +qa | 2024-05-10T00:17:00.000Z | 10.398003 +qa | 2024-05-10T00:08:00.000Z | 10.114583 +staging | 2024-05-10T00:20:00.000Z | 7.910156 +staging | 2024-05-10T00:13:00.000Z | 7.355903 +prod | 2024-05-10T00:02:00.000Z | 6.570313 +prod | 2024-05-10T00:18:00.000Z | 5.157552 + ; stddev_sq_and_var_over_time_are_consistent @@ -648,20 +651,23 @@ required_capability: variance_stddev_over_time TS k8s | STATS sd=max(stddev_over_time(network.cost)), var=max(stdvar_over_time(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) | EVAL sd_squared = sd * sd +| EVAL sd_squared = ROUND(sd_squared, 6), var = ROUND(var, 6), sd = ROUND(sd, 6) | KEEP sd, var, sd_squared, cluster, time_bucket | SORT cluster, time_bucket | LIMIT 10; -sd:double | var:double | sd_squared:double | cluster:keyword | time_bucket:datetime -1.4375 | 2.06640625 | 2.06640625 | prod | 2024-05-10T00:00:00.000Z -0.0 | 0.0 | 0.0 | prod | 2024-05-10T00:01:00.000Z -3.625 | 13.140625 | 13.140625 | prod | 2024-05-10T00:02:00.000Z -0.0 | 0.0 | 0.0 | prod | 2024-05-10T00:03:00.000Z -0.0 | 0.0 | 0.0 | prod | 2024-05-10T00:04:00.000Z -2.75 | 7.5625 | 7.5625 | prod | 2024-05-10T00:05:00.000Z -0.0 | 0.0 | 0.0 | prod | 2024-05-10T00:06:00.000Z -1.9375 | 3.75390625 | 3.75390625 | prod | 2024-05-10T00:08:00.000Z -3.406998516126605 | 11.607638888888891 | 11.60763888888889 | prod | 2024-05-10T00:09:00.000Z -0.0 | 0.0 | 0.0 | prod | 2024-05-10T00:10:00.000Z +sd:double | var:double | sd_squared:double | cluster:keyword | time_bucket:datetime +1.4375 | 2.066406 | 2.066406 | prod | 2024-05-10T00:00:00.000Z +0.0 | 0.0 | 0.0 | prod | 2024-05-10T00:01:00.000Z +3.625 | 13.140625 | 13.140625 | prod | 2024-05-10T00:02:00.000Z +0.0 | 0.0 | 0.0 | prod | 2024-05-10T00:03:00.000Z +0.0 | 0.0 | 0.0 | prod | 2024-05-10T00:04:00.000Z +2.75 | 7.5625 | 7.5625 | prod | 2024-05-10T00:05:00.000Z +0.0 | 0.0 | 0.0 | prod | 2024-05-10T00:06:00.000Z +1.9375 | 3.753906 | 3.753906 | prod | 2024-05-10T00:08:00.000Z +3.406999 | 11.607639 | 11.607639 | prod | 2024-05-10T00:09:00.000Z +0.0 | 0.0 | 0.0 | prod | 2024-05-10T00:10:00.000Z + + ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index 023bfd0076d78..07e4102550fa5 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -2994,6 +2994,18 @@ FROM employees | EVAL stddev_salary_change = ROUND(stddev_salary_change, 5) ; +stdVarianceNested +required_capability: variance_stddev_over_time + +FROM employees +| STATS stdvar_salary_change = STD_VAR(MV_MAX(salary_change) +| EVAL stdvar_salary_change = ROUND(stdvar_salary_change, 5) +; + +stdvar_salary_change:double +47.29269 +; + // tag::docsStatsStdDevNestedExpression-result[] stddev_salary_change:double 6.87583 @@ -3012,6 +3024,17 @@ std_dev:double 5.7601043E7 ; +stdVarianceWithLongs +required_capability: variance_stddev_over_time +FROM employees +| STATS std_var = STD_VAR(avg_worked_seconds) +| EVAL std_var = ROUND(std_var, 0) +; + +std_var:double +3.3179235E15 +; + stdDeviationWithInts required_capability: std_dev FROM employees @@ -3023,6 +3046,17 @@ std_dev:double 13765.1255 ; +stdVarianceWithInts +required_capability: variance_stddev_over_time +FROM employees +| STATS std_var = STD_VAR(salary) +| EVAL std_var = ROUND(std_var, 5) +; + +std_var:double +189493870.63564 +; + stdDeviationConstantValue required_capability: std_dev FROM employees @@ -3075,10 +3109,10 @@ stdDeviationNoRows required_capability: std_dev FROM employees | WHERE languages IS null -| STATS STD_DEV(languages) +| STATS sd=STD_DEV(languages) ; -STD_DEV(languages):double +sd:double null ; From 2278efc10ae878275f152d0ea11dc2557adce76d Mon Sep 17 00:00:00 2001 From: Pablo Date: Fri, 17 Oct 2025 16:08:16 -0700 Subject: [PATCH 17/24] fixup --- .../src/main/resources/stats.csv-spec | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index 07e4102550fa5..0dd3009037663 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -2994,25 +2994,25 @@ FROM employees | EVAL stddev_salary_change = ROUND(stddev_salary_change, 5) ; +// tag::docsStatsStdDevNestedExpression-result[] +stddev_salary_change:double +6.87583 +// end::docsStatsStdDevNestedExpression-result[] +; + + stdVarianceNested required_capability: variance_stddev_over_time FROM employees -| STATS stdvar_salary_change = STD_VAR(MV_MAX(salary_change) +| STATS stdvar_salary_change = STD_VAR(MV_MAX(salary_change)) | EVAL stdvar_salary_change = ROUND(stdvar_salary_change, 5) ; stdvar_salary_change:double -47.29269 +47.27703 ; -// tag::docsStatsStdDevNestedExpression-result[] -stddev_salary_change:double -6.87583 -// end::docsStatsStdDevNestedExpression-result[] -; - - stdDeviationWithLongs required_capability: std_dev FROM employees @@ -3028,11 +3028,11 @@ stdVarianceWithLongs required_capability: variance_stddev_over_time FROM employees | STATS std_var = STD_VAR(avg_worked_seconds) -| EVAL std_var = ROUND(std_var, 0) +| EVAL std_var = ROUND(std_var, 7) ; std_var:double -3.3179235E15 +3.317880108280233E15 ; stdDeviationWithInts @@ -3054,7 +3054,7 @@ FROM employees ; std_var:double -189493870.63564 +1.894786801075E8 ; stdDeviationConstantValue From 0106cf6ba928fefa46922bf0662f88d0bda64222 Mon Sep 17 00:00:00 2001 From: Pablo Date: Fri, 17 Oct 2025 17:53:52 -0700 Subject: [PATCH 18/24] regen docs --- .../_snippets/functions/examples/stdvar_over_time.md | 11 +++++------ .../src/main/resources/k8s-timeseries.csv-spec | 2 ++ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/stdvar_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/examples/stdvar_over_time.md index 8639e7b16d9a2..8ccfdf60f4400 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/examples/stdvar_over_time.md +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/stdvar_over_time.md @@ -7,12 +7,11 @@ TS k8s | STATS avg_stdvar_cost=AVG(STDVAR_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) ``` -| avg_stdvar_cost:double | cluster:keyword | time_bucket:datetime | +| cluster:keyword | time_bucket:datetime | avg_stdvar_cost:double | | --- | --- | --- | -| 20.478515625 | staging | 2024-05-10T00:03:00.000Z | -| 16.0 | qa | 2024-05-10T00:21:00.000Z | -| 11.192274305555557 | qa | 2024-05-10T00:18:00.000Z | -| 10.446903935185185 | staging | 2024-05-10T00:09:00.000Z | -| 10.398003472222221 | qa | 2024-05-10T00:17:00.000Z | +| staging | 2024-05-10T00:03:00.000Z | 20.478516 | +| qa | 2024-05-10T00:21:00.000Z | 16.0 | +| qa | 2024-05-10T00:18:00.000Z | 11.192274 | +| staging | 2024-05-10T00:09:00.000Z | 10.446904 | diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec index 6ab48109b533d..3fb629b40e964 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec @@ -629,12 +629,14 @@ TS k8s // end::stdvar_over_time[] | EVAL avg_stdvar_cost = ROUND(avg_stdvar_cost, 6) | SORT avg_stdvar_cost DESC, time_bucket DESC, cluster | LIMIT 10; + // tag::stdvar_over_time-result[] cluster:keyword | time_bucket:datetime | avg_stdvar_cost:double staging | 2024-05-10T00:03:00.000Z | 20.478516 qa | 2024-05-10T00:21:00.000Z | 16.0 qa | 2024-05-10T00:18:00.000Z | 11.192274 staging | 2024-05-10T00:09:00.000Z | 10.446904 +// end::stdvar_over_time-result[] qa | 2024-05-10T00:17:00.000Z | 10.398003 qa | 2024-05-10T00:08:00.000Z | 10.114583 staging | 2024-05-10T00:20:00.000Z | 7.910156 From 47e603e47d882d9b40392f77790cd546ae8cf0e7 Mon Sep 17 00:00:00 2001 From: Pablo Date: Sat, 18 Oct 2025 09:55:47 -0700 Subject: [PATCH 19/24] standardizing on name 'variance' with 'stdvar' as alias --- .../functions/examples/stddev_over_time.md | 13 ++++----- .../functions/examples/stdvar_over_time.md | 4 +-- .../images/functions/stdvar_over_time.svg | 2 +- .../functions/stdvar_over_time.json | 2 +- .../kibana/docs/functions/stdvar_over_time.md | 2 +- .../main/resources/k8s-timeseries.csv-spec | 10 +++---- .../src/main/resources/stats.csv-spec | 6 ++-- .../function/EsqlFunctionRegistry.java | 8 +++--- .../aggregate/AggregateWritables.java | 4 +-- .../aggregate/{StdVar.java => Variance.java} | 24 ++++++++-------- ...VarOverTime.java => VarianceOverTime.java} | 28 +++++++++---------- .../aggregate/StdvarOverTimeTests.java | 4 +-- .../{StdVarTests.java => VarianceTests.java} | 8 +++--- 13 files changed, 57 insertions(+), 58 deletions(-) rename x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/{StdVar.java => Variance.java} (79%) rename x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/{StdVarOverTime.java => VarianceOverTime.java} (74%) rename x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/{StdVarTests.java => VarianceTests.java} (89%) diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/stddev_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/examples/stddev_over_time.md index 20780fdca810b..022d6dfa95f42 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/examples/stddev_over_time.md +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/stddev_over_time.md @@ -7,13 +7,12 @@ TS k8s | STATS max_stddev_cost=MAX(STDDEV_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) ``` -| max_stddev_cost:double | cluster:keyword | time_bucket:datetime | +| cluster:keyword | time_bucket:datetime | max_stddev_cost:double | | --- | --- | --- | -| 5.4375 | staging | 2024-05-10T00:03:00.000Z | -| 5.1875 | staging | 2024-05-10T00:09:00.000Z | -| 4.097763617714749 | qa | 2024-05-10T00:18:00.000Z | -| 4.0 | qa | 2024-05-10T00:21:00.000Z | -| 3.9375 | staging | 2024-05-10T00:20:00.000Z | -| 3.9335297443898907 | prod | 2024-05-10T00:18:00.000Z | +| staging | 2024-05-10T00:03:00.000Z | 5.4375 | +| staging | 2024-05-10T00:09:00.000Z | 5.1875 | +| qa | 2024-05-10T00:18:00.000Z | 4.097764 | +| qa | 2024-05-10T00:21:00.000Z | 4.0 | +| staging | 2024-05-10T00:20:00.000Z | 3.9375 | diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/stdvar_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/examples/stdvar_over_time.md index 8ccfdf60f4400..1aa98d1f64e28 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/examples/stdvar_over_time.md +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/stdvar_over_time.md @@ -4,10 +4,10 @@ ```esql TS k8s -| STATS avg_stdvar_cost=AVG(STDVAR_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) +| STATS avg_var_cost=AVG(VARIANCE_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) ``` -| cluster:keyword | time_bucket:datetime | avg_stdvar_cost:double | +| cluster:keyword | time_bucket:datetime | avg_var_cost:double | | --- | --- | --- | | staging | 2024-05-10T00:03:00.000Z | 20.478516 | | qa | 2024-05-10T00:21:00.000Z | 16.0 | diff --git a/docs/reference/query-languages/esql/images/functions/stdvar_over_time.svg b/docs/reference/query-languages/esql/images/functions/stdvar_over_time.svg index 0c0e256a06780..ebef8c3c83310 100644 --- a/docs/reference/query-languages/esql/images/functions/stdvar_over_time.svg +++ b/docs/reference/query-languages/esql/images/functions/stdvar_over_time.svg @@ -1 +1 @@ -STDVAR_OVER_TIME(number) \ No newline at end of file +VARIANCE_OVER_TIME(number) \ No newline at end of file diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/stdvar_over_time.json b/docs/reference/query-languages/esql/kibana/definition/functions/stdvar_over_time.json index 41f1d968e1e74..a5e32548998f9 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/stdvar_over_time.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/stdvar_over_time.json @@ -42,7 +42,7 @@ } ], "examples" : [ - "TS k8s\n| STATS avg_stdvar_cost=AVG(STDVAR_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute)" + "TS k8s\n| STATS avg_var_cost=AVG(VARIANCE_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute)" ], "preview" : true, "snapshot_only" : false diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/stdvar_over_time.md b/docs/reference/query-languages/esql/kibana/docs/functions/stdvar_over_time.md index 46ea25b048b70..23d69f15d7be3 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/stdvar_over_time.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/stdvar_over_time.md @@ -5,5 +5,5 @@ Calculates the population standard variance over time of a numeric field. ```esql TS k8s -| STATS avg_stdvar_cost=AVG(STDVAR_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) +| STATS avg_var_cost=AVG(VARIANCE_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) ``` diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec index 3fb629b40e964..a60f031ba2b58 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec @@ -610,8 +610,8 @@ staging | 2024-05-10T00:03:00.000Z | 5.4375 staging | 2024-05-10T00:09:00.000Z | 5.1875 qa | 2024-05-10T00:18:00.000Z | 4.097764 qa | 2024-05-10T00:21:00.000Z | 4.0 -// end::stddev_over_time-result[] staging | 2024-05-10T00:20:00.000Z | 3.9375 +// end::stddev_over_time-result[] prod | 2024-05-10T00:18:00.000Z | 3.93353 qa | 2024-05-10T00:08:00.000Z | 3.89444 qa | 2024-05-10T00:17:00.000Z | 3.882385 @@ -625,13 +625,13 @@ required_capability: ts_command_v0 required_capability: variance_stddev_over_time // tag::stdvar_over_time[] TS k8s -| STATS avg_stdvar_cost=AVG(STDVAR_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) +| STATS avg_var_cost=AVG(VARIANCE_OVER_TIME(network.cost)) BY cluster, time_bucket = TBUCKET(1minute) // end::stdvar_over_time[] -| EVAL avg_stdvar_cost = ROUND(avg_stdvar_cost, 6) -| SORT avg_stdvar_cost DESC, time_bucket DESC, cluster | LIMIT 10; +| EVAL avg_var_cost = ROUND(avg_var_cost, 6) +| SORT avg_var_cost DESC, time_bucket DESC, cluster | LIMIT 10; // tag::stdvar_over_time-result[] -cluster:keyword | time_bucket:datetime | avg_stdvar_cost:double +cluster:keyword | time_bucket:datetime | avg_var_cost:double staging | 2024-05-10T00:03:00.000Z | 20.478516 qa | 2024-05-10T00:21:00.000Z | 16.0 qa | 2024-05-10T00:18:00.000Z | 11.192274 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index 0dd3009037663..e22c87d1d9eb3 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -3027,11 +3027,11 @@ std_dev:double stdVarianceWithLongs required_capability: variance_stddev_over_time FROM employees -| STATS std_var = STD_VAR(avg_worked_seconds) -| EVAL std_var = ROUND(std_var, 7) +| STATS var = variance(avg_worked_seconds) +| EVAL var = ROUND(var, 7) ; -std_var:double +var:double 3.317880108280233E15 ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 10121f1aa2ce8..6b79e90082fdc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -50,12 +50,12 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialExtent; import org.elasticsearch.xpack.esql.expression.function.aggregate.StdDev; import org.elasticsearch.xpack.esql.expression.function.aggregate.StdDevOverTime; -import org.elasticsearch.xpack.esql.expression.function.aggregate.StdVar; -import org.elasticsearch.xpack.esql.expression.function.aggregate.StdVarOverTime; import org.elasticsearch.xpack.esql.expression.function.aggregate.Sum; import org.elasticsearch.xpack.esql.expression.function.aggregate.SumOverTime; import org.elasticsearch.xpack.esql.expression.function.aggregate.Top; import org.elasticsearch.xpack.esql.expression.function.aggregate.Values; +import org.elasticsearch.xpack.esql.expression.function.aggregate.Variance; +import org.elasticsearch.xpack.esql.expression.function.aggregate.VarianceOverTime; import org.elasticsearch.xpack.esql.expression.function.aggregate.WeightedAvg; import org.elasticsearch.xpack.esql.expression.function.fulltext.Kql; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; @@ -358,7 +358,7 @@ private static FunctionDefinition[][] functions() { def(Percentile.class, bi(Percentile::new), "percentile"), def(Sample.class, bi(Sample::new), "sample"), def(StdDev.class, uni(StdDev::new), "std_dev"), - def(StdVar.class, uni(StdVar::new), "std_var"), + def(Variance.class, uni(Variance::new), "variance", "std_var"), def(Sum.class, uni(Sum::new), "sum"), def(Top.class, tri(Top::new), "top"), def(Values.class, uni(Values::new), "values"), @@ -532,7 +532,7 @@ private static FunctionDefinition[][] functions() { def(MinOverTime.class, uni(MinOverTime::new), "min_over_time"), def(SumOverTime.class, uni(SumOverTime::new), "sum_over_time"), def(StdDevOverTime.class, uni(StdDevOverTime::new), "stddev_over_time"), - def(StdVarOverTime.class, uni(StdVarOverTime::new), "stdvar_over_time"), + def(VarianceOverTime.class, uni(VarianceOverTime::new), "variance_over_time", "stdvar_over_time"), def(CountOverTime.class, uni(CountOverTime::new), "count_over_time"), def(CountDistinctOverTime.class, bi(CountDistinctOverTime::new), "count_distinct_over_time"), def(PresentOverTime.class, uni(PresentOverTime::new), "present_over_time"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateWritables.java index 00927e71fe9eb..727238ca524c4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateWritables.java @@ -33,7 +33,7 @@ public static List getNamedWriteables() { SpatialCentroid.ENTRY, SpatialExtent.ENTRY, StdDev.ENTRY, - StdVar.ENTRY, + Variance.ENTRY, Sum.ENTRY, Top.ENTRY, Values.ENTRY, @@ -41,7 +41,7 @@ public static List getNamedWriteables() { MaxOverTime.ENTRY, AvgOverTime.ENTRY, StdDevOverTime.ENTRY, - StdVarOverTime.ENTRY, + VarianceOverTime.ENTRY, Last.ENTRY, LastOverTime.ENTRY, FirstOverTime.ENTRY, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVar.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Variance.java similarity index 79% rename from x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVar.java rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Variance.java index a02bc9c60e53e..9caa5714fc0ec 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVar.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Variance.java @@ -32,24 +32,24 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; -public class StdVar extends AggregateFunction implements ToAggregator { - public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "StdVar", StdVar::new); +public class Variance extends AggregateFunction implements ToAggregator { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Variance", Variance::new); @FunctionInfo( returnType = "double", description = "The population standard variance of a numeric field.", type = FunctionType.AGGREGATE, - examples = { @Example(file = "stats", tag = "stdev") } + examples = { @Example(file = "stats", tag = "stdvar") } ) - public StdVar(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) { + public Variance(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) { this(source, field, Literal.TRUE); } - public StdVar(Source source, Expression field, Expression filter) { + public Variance(Source source, Expression field, Expression filter) { super(source, field, filter, emptyList()); } - private StdVar(StreamInput in) throws IOException { + private Variance(StreamInput in) throws IOException { super(in); } @@ -75,17 +75,17 @@ protected Expression.TypeResolution resolveType() { } @Override - protected NodeInfo info() { - return NodeInfo.create(this, StdVar::new, field(), filter()); + protected NodeInfo info() { + return NodeInfo.create(this, Variance::new, field(), filter()); } @Override - public StdVar replaceChildren(List newChildren) { - return new StdVar(source(), newChildren.get(0), newChildren.get(1)); + public Variance replaceChildren(List newChildren) { + return new Variance(source(), newChildren.get(0), newChildren.get(1)); } - public StdVar withFilter(Expression filter) { - return new StdVar(source(), field(), filter); + public Variance withFilter(Expression filter) { + return new Variance(source(), field(), filter); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/VarianceOverTime.java similarity index 74% rename from x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTime.java rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/VarianceOverTime.java index a7cb78f330e3c..ac0ee30f7e569 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/VarianceOverTime.java @@ -27,13 +27,13 @@ import static java.util.Collections.emptyList; /** - * Similar to {@link StdVar}, but it is used to calculate the standard deviation over a time series of values from the given field. + * Similar to {@link Variance}, but it is used to calculate the standard deviation over a time series of values from the given field. */ -public class StdVarOverTime extends TimeSeriesAggregateFunction { +public class VarianceOverTime extends TimeSeriesAggregateFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, - "StdVarOverTime", - StdVarOverTime::new + "VarianceOverTime", + VarianceOverTime::new ); @FunctionInfo( @@ -44,7 +44,7 @@ public class StdVarOverTime extends TimeSeriesAggregateFunction { preview = true, examples = { @Example(file = "k8s-timeseries", tag = "stdvar_over_time") } ) - public StdVarOverTime( + public VarianceOverTime( Source source, @Param( name = "number", @@ -55,11 +55,11 @@ public StdVarOverTime( this(source, field, Literal.TRUE); } - public StdVarOverTime(Source source, Expression field, Expression filter) { + public VarianceOverTime(Source source, Expression field, Expression filter) { super(source, field, filter, emptyList()); } - private StdVarOverTime(StreamInput in) throws IOException { + private VarianceOverTime(StreamInput in) throws IOException { super(in); } @@ -79,22 +79,22 @@ public DataType dataType() { } @Override - protected NodeInfo info() { - return NodeInfo.create(this, StdVarOverTime::new, field(), filter()); + protected NodeInfo info() { + return NodeInfo.create(this, VarianceOverTime::new, field(), filter()); } @Override - public StdVarOverTime replaceChildren(List newChildren) { - return new StdVarOverTime(source(), newChildren.get(0), newChildren.get(1)); + public VarianceOverTime replaceChildren(List newChildren) { + return new VarianceOverTime(source(), newChildren.get(0), newChildren.get(1)); } @Override - public StdVarOverTime withFilter(Expression filter) { - return new StdVarOverTime(source(), field(), filter); + public VarianceOverTime withFilter(Expression filter) { + return new VarianceOverTime(source(), field(), filter); } @Override public AggregateFunction perTimeSeriesAggregation() { - return new StdVar(source(), field(), filter()); + return new Variance(source(), field(), filter()); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdvarOverTimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdvarOverTimeTests.java index 1ee1b478cce05..c9911a041c849 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdvarOverTimeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdvarOverTimeTests.java @@ -24,11 +24,11 @@ public StdvarOverTimeTests(Supplier testCaseSupplier) @ParametersFactory public static Iterable parameters() { - return StdVarTests.parameters(); + return VarianceTests.parameters(); } @Override protected Expression build(Source source, List args) { - return new StdVarOverTime(source, args.get(0)); + return new VarianceOverTime(source, args.get(0)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/VarianceTests.java similarity index 89% rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/VarianceTests.java index 300705bbb3eec..99b853b24c3d2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdVarTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/VarianceTests.java @@ -26,8 +26,8 @@ import static org.hamcrest.Matchers.equalTo; -public class StdVarTests extends AbstractAggregationTestCase { - public StdVarTests(@Name("TestCase") Supplier testCaseSupplier) { +public class VarianceTests extends AbstractAggregationTestCase { + public VarianceTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @@ -39,14 +39,14 @@ public static Iterable parameters() { MultiRowTestCaseSupplier.intCases(1, 1000, Integer.MIN_VALUE, Integer.MAX_VALUE, true), MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true), MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true) - ).flatMap(List::stream).map(StdVarTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); + ).flatMap(List::stream).map(VarianceTests::makeSupplier).collect(Collectors.toCollection(() -> suppliers)); return parameterSuppliersFromTypedDataWithDefaultChecks(suppliers, true); } @Override protected Expression build(Source source, List args) { - return new StdVar(source, args.get(0)); + return new Variance(source, args.get(0)); } private static TestCaseSupplier makeSupplier(TestCaseSupplier.TypedDataSupplier fieldSupplier) { From 8179b6772f1bfd458c85bb2f62deb1a7955d4e2e Mon Sep 17 00:00:00 2001 From: Pablo Date: Sat, 18 Oct 2025 10:45:11 -0700 Subject: [PATCH 20/24] variance docs --- .../functions/description/variance.md | 6 +++ .../_snippets/functions/examples/variance.md | 9 ++++ .../_snippets/functions/layout/variance.md | 23 +++++++++ .../functions/parameters/variance.md | 7 +++ .../_snippets/functions/types/variance.md | 10 ++++ .../esql/images/functions/variance.svg | 1 + .../kibana/definition/functions/variance.json | 49 +++++++++++++++++++ .../esql/kibana/docs/functions/variance.md | 8 +++ 8 files changed, 113 insertions(+) create mode 100644 docs/reference/query-languages/esql/_snippets/functions/description/variance.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/examples/variance.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/layout/variance.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/parameters/variance.md create mode 100644 docs/reference/query-languages/esql/_snippets/functions/types/variance.md create mode 100644 docs/reference/query-languages/esql/images/functions/variance.svg create mode 100644 docs/reference/query-languages/esql/kibana/definition/functions/variance.json create mode 100644 docs/reference/query-languages/esql/kibana/docs/functions/variance.md diff --git a/docs/reference/query-languages/esql/_snippets/functions/description/variance.md b/docs/reference/query-languages/esql/_snippets/functions/description/variance.md new file mode 100644 index 0000000000000..0772559a8a5d9 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/description/variance.md @@ -0,0 +1,6 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Description** + +The population standard variance of a numeric field. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/variance.md b/docs/reference/query-languages/esql/_snippets/functions/examples/variance.md new file mode 100644 index 0000000000000..b3efac50b7ae7 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/variance.md @@ -0,0 +1,9 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Example** + +```esql +null +``` + + diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/variance.md b/docs/reference/query-languages/esql/_snippets/functions/layout/variance.md new file mode 100644 index 0000000000000..4db08b9e8caae --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/layout/variance.md @@ -0,0 +1,23 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +## `VARIANCE` [esql-variance] + +**Syntax** + +:::{image} ../../../images/functions/variance.svg +:alt: Embedded +:class: text-center +::: + + +:::{include} ../parameters/variance.md +::: + +:::{include} ../description/variance.md +::: + +:::{include} ../types/variance.md +::: + +:::{include} ../examples/variance.md +::: diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/variance.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/variance.md new file mode 100644 index 0000000000000..e003b3562deeb --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/parameters/variance.md @@ -0,0 +1,7 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Parameters** + +`number` +: + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/variance.md b/docs/reference/query-languages/esql/_snippets/functions/types/variance.md new file mode 100644 index 0000000000000..e204309f17bb1 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/variance.md @@ -0,0 +1,10 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| number | result | +| --- | --- | +| double | double | +| integer | double | +| long | double | + diff --git a/docs/reference/query-languages/esql/images/functions/variance.svg b/docs/reference/query-languages/esql/images/functions/variance.svg new file mode 100644 index 0000000000000..08eb2333082fc --- /dev/null +++ b/docs/reference/query-languages/esql/images/functions/variance.svg @@ -0,0 +1 @@ +VARIANCE(number) \ No newline at end of file diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/variance.json b/docs/reference/query-languages/esql/kibana/definition/functions/variance.json new file mode 100644 index 0000000000000..227ba65da776a --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/functions/variance.json @@ -0,0 +1,49 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "agg", + "name" : "variance", + "description" : "The population standard variance of a numeric field.", + "signatures" : [ + { + "params" : [ + { + "name" : "number", + "type" : "double", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "integer", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "long", + "optional" : false, + "description" : "" + } + ], + "variadic" : false, + "returnType" : "double" + } + ], + "examples" : [ + null + ], + "preview" : false, + "snapshot_only" : false +} diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/variance.md b/docs/reference/query-languages/esql/kibana/docs/functions/variance.md new file mode 100644 index 0000000000000..7f54526d0339e --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/docs/functions/variance.md @@ -0,0 +1,8 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +### VARIANCE +The population standard variance of a numeric field. + +```esql +null +``` From cc93e9d5a0ba89154606a243467f77d52f5312aa Mon Sep 17 00:00:00 2001 From: Pablo Date: Mon, 20 Oct 2025 09:56:18 -0700 Subject: [PATCH 21/24] oops removing unneeded file --- .swp | Bin 12288 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .swp diff --git a/.swp b/.swp deleted file mode 100644 index 1269dcf1d1617b7151d88e591c91282820eb4ad9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI&%T60H6b4|2T{jhi7dUO#83WyvU9&-oDpVGMz2lq?hPlPZ6Gd#0`V6QK)<@t; zI71R{N_Hr0eoG!_Y-`T%Z#SBZ4=#qMbetU;ZM)rN(W;A{Y23J|%hk;0aj~z@g_9Kg z=6zU!00cf#;M4i{qr-zP!MCqpXn*hgBg3FJ1Rwwb2tWV=5P$##Ah2bD)bz})*4$av z?QIyh91G$>d z4FV8=00bZa0SG_<0uX=z1R$^wxWD^L+WxeaPob{JEGIV2|D5+5=dW#3w0Wp3%e1gl zXA?^rmQBGWSBcIzdL2W-zh@jrq2%n!%^Z!L@PxF=x~E9=wsiW;ZmX?hZkPW}7k1HK z+WBMM$AUa>B8zBVy9#byo=47IVmp}SugYdLtbEZfyyLQ)r(yr!YBKFsn+Ct;pi zEoGm==}q10>BL_Dkbcsm3CF;ad71U4VNdDOC2ECb@oN<@qd_4x`N*1ajnGIV$s48@ KCwX4Z$UFdP6|}kl From 7bdf4971da34c076f49ee0ede8906fd1236445f5 Mon Sep 17 00:00:00 2001 From: Pablo Date: Mon, 20 Oct 2025 20:24:46 -0700 Subject: [PATCH 22/24] comments --- .../functions/description/std_var.md | 6 ----- .../_snippets/functions/examples/std_var.md | 14 ----------- .../_snippets/functions/layout/std_var.md | 23 ------------------- .../_snippets/functions/parameters/std_var.md | 7 ------ .../esql/_snippets/functions/types/std_var.md | 10 -------- .../function/aggregate/Variance.java | 2 +- .../function/aggregate/VarianceOverTime.java | 4 ++-- 7 files changed, 3 insertions(+), 63 deletions(-) delete mode 100644 docs/reference/query-languages/esql/_snippets/functions/description/std_var.md delete mode 100644 docs/reference/query-languages/esql/_snippets/functions/examples/std_var.md delete mode 100644 docs/reference/query-languages/esql/_snippets/functions/layout/std_var.md delete mode 100644 docs/reference/query-languages/esql/_snippets/functions/parameters/std_var.md delete mode 100644 docs/reference/query-languages/esql/_snippets/functions/types/std_var.md diff --git a/docs/reference/query-languages/esql/_snippets/functions/description/std_var.md b/docs/reference/query-languages/esql/_snippets/functions/description/std_var.md deleted file mode 100644 index 0772559a8a5d9..0000000000000 --- a/docs/reference/query-languages/esql/_snippets/functions/description/std_var.md +++ /dev/null @@ -1,6 +0,0 @@ -% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. - -**Description** - -The population standard variance of a numeric field. - diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/std_var.md b/docs/reference/query-languages/esql/_snippets/functions/examples/std_var.md deleted file mode 100644 index d10976b1794d3..0000000000000 --- a/docs/reference/query-languages/esql/_snippets/functions/examples/std_var.md +++ /dev/null @@ -1,14 +0,0 @@ -% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. - -**Example** - -```esql -FROM employees -| STATS std_dev_height = STD_DEV(height) -``` - -| std_dev_height:double | -| --- | -| 0.2063704 | - - diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/std_var.md b/docs/reference/query-languages/esql/_snippets/functions/layout/std_var.md deleted file mode 100644 index d19ab9ce11eb6..0000000000000 --- a/docs/reference/query-languages/esql/_snippets/functions/layout/std_var.md +++ /dev/null @@ -1,23 +0,0 @@ -% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. - -## `STD_VAR` [esql-std_var] - -**Syntax** - -:::{image} ../../../images/functions/std_var.svg -:alt: Embedded -:class: text-center -::: - - -:::{include} ../parameters/std_var.md -::: - -:::{include} ../description/std_var.md -::: - -:::{include} ../types/std_var.md -::: - -:::{include} ../examples/std_var.md -::: diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/std_var.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/std_var.md deleted file mode 100644 index e003b3562deeb..0000000000000 --- a/docs/reference/query-languages/esql/_snippets/functions/parameters/std_var.md +++ /dev/null @@ -1,7 +0,0 @@ -% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. - -**Parameters** - -`number` -: - diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/std_var.md b/docs/reference/query-languages/esql/_snippets/functions/types/std_var.md deleted file mode 100644 index e204309f17bb1..0000000000000 --- a/docs/reference/query-languages/esql/_snippets/functions/types/std_var.md +++ /dev/null @@ -1,10 +0,0 @@ -% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. - -**Supported types** - -| number | result | -| --- | --- | -| double | double | -| integer | double | -| long | double | - diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Variance.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Variance.java index 9caa5714fc0ec..2f6a9595d0c05 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Variance.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Variance.java @@ -37,7 +37,7 @@ public class Variance extends AggregateFunction implements ToAggregator { @FunctionInfo( returnType = "double", - description = "The population standard variance of a numeric field.", + description = "The population variance of a numeric field.", type = FunctionType.AGGREGATE, examples = { @Example(file = "stats", tag = "stdvar") } ) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/VarianceOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/VarianceOverTime.java index ac0ee30f7e569..440dabd73cb40 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/VarianceOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/VarianceOverTime.java @@ -27,7 +27,7 @@ import static java.util.Collections.emptyList; /** - * Similar to {@link Variance}, but it is used to calculate the standard deviation over a time series of values from the given field. + * Similar to {@link Variance}, but it is used to calculate the variance over a time series of values from the given field. */ public class VarianceOverTime extends TimeSeriesAggregateFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( @@ -38,7 +38,7 @@ public class VarianceOverTime extends TimeSeriesAggregateFunction { @FunctionInfo( returnType = "double", - description = "Calculates the population standard variance over time of a numeric field.", + description = "Calculates the population variance over time of a numeric field.", type = FunctionType.TIME_SERIES_AGGREGATE, appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.3.0") }, preview = true, From b4aeae86e9fe7a91636cf89b11f68a84a3c1bcbd Mon Sep 17 00:00:00 2001 From: Pablo Date: Tue, 21 Oct 2025 10:29:19 -0700 Subject: [PATCH 23/24] regen docs --- .../functions/description/stdvar_over_time.md | 2 +- .../functions/description/variance.md | 2 +- .../esql/images/functions/std_var.svg | 1 - .../kibana/definition/functions/std_var.json | 49 ------------------- .../functions/stdvar_over_time.json | 2 +- .../kibana/definition/functions/variance.json | 2 +- .../esql/kibana/docs/functions/std_var.md | 9 ---- .../kibana/docs/functions/stdvar_over_time.md | 2 +- .../esql/kibana/docs/functions/variance.md | 2 +- 9 files changed, 6 insertions(+), 65 deletions(-) delete mode 100644 docs/reference/query-languages/esql/images/functions/std_var.svg delete mode 100644 docs/reference/query-languages/esql/kibana/definition/functions/std_var.json delete mode 100644 docs/reference/query-languages/esql/kibana/docs/functions/std_var.md diff --git a/docs/reference/query-languages/esql/_snippets/functions/description/stdvar_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/description/stdvar_over_time.md index 67b3a64b7bceb..16ec531dba29f 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/description/stdvar_over_time.md +++ b/docs/reference/query-languages/esql/_snippets/functions/description/stdvar_over_time.md @@ -2,5 +2,5 @@ **Description** -Calculates the population standard variance over time of a numeric field. +Calculates the population variance over time of a numeric field. diff --git a/docs/reference/query-languages/esql/_snippets/functions/description/variance.md b/docs/reference/query-languages/esql/_snippets/functions/description/variance.md index 0772559a8a5d9..d9b489b9037bd 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/description/variance.md +++ b/docs/reference/query-languages/esql/_snippets/functions/description/variance.md @@ -2,5 +2,5 @@ **Description** -The population standard variance of a numeric field. +The population variance of a numeric field. diff --git a/docs/reference/query-languages/esql/images/functions/std_var.svg b/docs/reference/query-languages/esql/images/functions/std_var.svg deleted file mode 100644 index 53f007458df97..0000000000000 --- a/docs/reference/query-languages/esql/images/functions/std_var.svg +++ /dev/null @@ -1 +0,0 @@ -STD_VAR(number) \ No newline at end of file diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/std_var.json b/docs/reference/query-languages/esql/kibana/definition/functions/std_var.json deleted file mode 100644 index 9da94c0adde88..0000000000000 --- a/docs/reference/query-languages/esql/kibana/definition/functions/std_var.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.", - "type" : "agg", - "name" : "std_var", - "description" : "The population standard variance of a numeric field.", - "signatures" : [ - { - "params" : [ - { - "name" : "number", - "type" : "double", - "optional" : false, - "description" : "" - } - ], - "variadic" : false, - "returnType" : "double" - }, - { - "params" : [ - { - "name" : "number", - "type" : "integer", - "optional" : false, - "description" : "" - } - ], - "variadic" : false, - "returnType" : "double" - }, - { - "params" : [ - { - "name" : "number", - "type" : "long", - "optional" : false, - "description" : "" - } - ], - "variadic" : false, - "returnType" : "double" - } - ], - "examples" : [ - "FROM employees\n| STATS std_dev_height = STD_DEV(height)" - ], - "preview" : false, - "snapshot_only" : false -} diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/stdvar_over_time.json b/docs/reference/query-languages/esql/kibana/definition/functions/stdvar_over_time.json index a5e32548998f9..b6312d700ab06 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/stdvar_over_time.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/stdvar_over_time.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.", "type" : "time_series_agg", "name" : "stdvar_over_time", - "description" : "Calculates the population standard variance over time of a numeric field.", + "description" : "Calculates the population variance over time of a numeric field.", "signatures" : [ { "params" : [ diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/variance.json b/docs/reference/query-languages/esql/kibana/definition/functions/variance.json index 227ba65da776a..e6cddc4f79d13 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/variance.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/variance.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.", "type" : "agg", "name" : "variance", - "description" : "The population standard variance of a numeric field.", + "description" : "The population variance of a numeric field.", "signatures" : [ { "params" : [ diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/std_var.md b/docs/reference/query-languages/esql/kibana/docs/functions/std_var.md deleted file mode 100644 index 8ab27cc6c1d1a..0000000000000 --- a/docs/reference/query-languages/esql/kibana/docs/functions/std_var.md +++ /dev/null @@ -1,9 +0,0 @@ -% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. - -### STD VAR -The population standard variance of a numeric field. - -```esql -FROM employees -| STATS std_dev_height = STD_DEV(height) -``` diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/stdvar_over_time.md b/docs/reference/query-languages/esql/kibana/docs/functions/stdvar_over_time.md index 23d69f15d7be3..ff1bea59007b8 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/stdvar_over_time.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/stdvar_over_time.md @@ -1,7 +1,7 @@ % This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. ### STDVAR OVER TIME -Calculates the population standard variance over time of a numeric field. +Calculates the population variance over time of a numeric field. ```esql TS k8s diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/variance.md b/docs/reference/query-languages/esql/kibana/docs/functions/variance.md index 7f54526d0339e..febfa44b540e6 100644 --- a/docs/reference/query-languages/esql/kibana/docs/functions/variance.md +++ b/docs/reference/query-languages/esql/kibana/docs/functions/variance.md @@ -1,7 +1,7 @@ % This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. ### VARIANCE -The population standard variance of a numeric field. +The population variance of a numeric field. ```esql null From d026609424abb851cc20f49f7b3c0ad1b3f1adb1 Mon Sep 17 00:00:00 2001 From: Pablo Date: Tue, 21 Oct 2025 13:30:35 -0700 Subject: [PATCH 24/24] check version influence on docs --- .../esql/_snippets/functions/layout/stddev_over_time.md | 2 +- .../esql/_snippets/functions/layout/stdvar_over_time.md | 2 +- .../esql/expression/function/aggregate/StdDevOverTime.java | 2 +- .../esql/expression/function/aggregate/VarianceOverTime.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/stddev_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/layout/stddev_over_time.md index e991631595071..99d2d639f4cf5 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/layout/stddev_over_time.md +++ b/docs/reference/query-languages/esql/_snippets/functions/layout/stddev_over_time.md @@ -2,7 +2,7 @@ ## `STDDEV_OVER_TIME` [esql-stddev_over_time] ```{applies_to} -stack: preview 9.3.0 +stack: preview 9.2.0 serverless: preview ``` diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/stdvar_over_time.md b/docs/reference/query-languages/esql/_snippets/functions/layout/stdvar_over_time.md index 29c6e73c1f785..24aa0696c3e4a 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/layout/stdvar_over_time.md +++ b/docs/reference/query-languages/esql/_snippets/functions/layout/stdvar_over_time.md @@ -2,7 +2,7 @@ ## `STDVAR_OVER_TIME` [esql-stdvar_over_time] ```{applies_to} -stack: preview 9.3.0 +stack: preview 9.2.0 serverless: preview ``` diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTime.java index 7f902e0c1acd7..262b6c71aa027 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/StdDevOverTime.java @@ -40,7 +40,7 @@ public class StdDevOverTime extends TimeSeriesAggregateFunction { returnType = "double", description = "Calculates the population standard deviation over time of a numeric field.", type = FunctionType.TIME_SERIES_AGGREGATE, - appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.3.0") }, + appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0") }, preview = true, examples = { @Example(file = "k8s-timeseries", tag = "stddev_over_time") } ) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/VarianceOverTime.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/VarianceOverTime.java index 440dabd73cb40..36821ce323580 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/VarianceOverTime.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/VarianceOverTime.java @@ -40,7 +40,7 @@ public class VarianceOverTime extends TimeSeriesAggregateFunction { returnType = "double", description = "Calculates the population variance over time of a numeric field.", type = FunctionType.TIME_SERIES_AGGREGATE, - appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.3.0") }, + appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.2.0") }, preview = true, examples = { @Example(file = "k8s-timeseries", tag = "stdvar_over_time") } )