From 43b1fb594c662399fdaa861c8a47c03da885950f Mon Sep 17 00:00:00 2001 From: gmarouli Date: Tue, 25 Nov 2025 11:48:42 +0200 Subject: [PATCH 1/8] Added missing capability to the downsample API and fixed broken yaml test --- .../rest-api-spec/test/downsample/10_basic.yml | 10 +++++----- .../xpack/downsample/RestDownsampleAction.java | 11 +++++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml index ca62f98c0ee00..74e572708cbab 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml @@ -346,7 +346,7 @@ setup: - match: { hits.hits.0._source.k8s\.pod\.values: [1, 1, 2] } - is_false: hits.hits.0._source.k8s\.pod\.running - # Assert rollup index settings + # Assert downsample index settings - do: indices.get_settings: index: test-downsample @@ -358,7 +358,7 @@ setup: - match: { test-downsample.settings.index.downsample.source.name: test } - match: { test-downsample.settings.index.number_of_shards: "1" } - # Assert rollup index mapping + # Assert downsample index mapping - do: indices.get_mapping: index: test-downsample @@ -432,7 +432,7 @@ setup: - match: { hits.hits.0._source.k8s\.pod\.values: [1, 1, 2] } - is_false: hits.hits.0._source.k8s\.pod\.running - # Assert rollup index settings + # Assert downsample index settings - do: indices.get_settings: index: test-downsample @@ -442,7 +442,7 @@ setup: - match: { test-downsample.settings.index.time_series.start_time: 2021-04-28T00:00:00Z } - match: { test-downsample.settings.index.routing_path: [ "metricset", "k8s.pod.uid"] } - match: { test-downsample.settings.index.downsample.source.name: test } - - match: { test-downsample.settings.index.downsample.method: last_value } + - match: { test-downsample.settings.index.downsample.sampling_method: last_value } - match: { test-downsample.settings.index.number_of_shards: "1" } # Assert rollup index mapping @@ -1001,7 +1001,7 @@ setup: - do: catch: /Downsampling method \[aggregate\] is not compatible with the source index downsampling method \[last_value\]/ indices.downsample: - index: test-downsample + index: test-downsample-last-value target_index: test-downsample-2 body: > { diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/RestDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/RestDownsampleAction.java index eb8dfe72850a2..ee8996249ca3b 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/RestDownsampleAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/RestDownsampleAction.java @@ -20,12 +20,19 @@ import java.io.IOException; import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.elasticsearch.rest.RestRequest.Method.POST; @ServerlessScope(Scope.INTERNAL) public class RestDownsampleAction extends BaseRestHandler { + private final Set CAPABILITIES = Stream.of( + "downsample.sampling_mode.last_value", + ).filter(Objects::nonNull).collect(Collectors.toSet()); @Override public List routes() { return List.of(new Route(POST, "/{index}/_downsample/{target_index}")); @@ -55,4 +62,8 @@ public String getName() { return "downsample_action"; } + @Override + public Set supportedCapabilities() { + return CAPABILITIES; + } } From eba62818dcf62124889506901e9d96f248febac1 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Tue, 25 Nov 2025 12:03:53 +0200 Subject: [PATCH 2/8] Rename MetricFieldProducer to NumericMetricFieldProducer --- .../AggregateMetricFieldSerializer.java | 4 +- .../AggregateSubMetricFieldValueFetcher.java | 2 +- .../downsample/DownsampleShardIndexer.java | 12 ++--- .../xpack/downsample/FieldValueFetcher.java | 2 +- ...r.java => NumericMetricFieldProducer.java} | 17 +++--- .../downsample/RestDownsampleAction.java | 7 +-- .../AggregateMetricFieldSerializerTests.java | 12 ++--- ...a => NumericMetricFieldProducerTests.java} | 54 +++++++++++++------ 8 files changed, 68 insertions(+), 42 deletions(-) rename x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/{MetricFieldProducer.java => NumericMetricFieldProducer.java} (89%) rename x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/{MetricFieldProducerTests.java => NumericMetricFieldProducerTests.java} (81%) diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldSerializer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldSerializer.java index da659af9ba2bd..881bc4511eee3 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldSerializer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldSerializer.java @@ -44,13 +44,13 @@ public void write(XContentBuilder builder) throws IOException { continue; } switch (fieldProducer) { - case MetricFieldProducer.AggregateGaugeMetricFieldProducer producer -> { + case NumericMetricFieldProducer.AggregateGaugeMetricFieldProducer producer -> { builder.field("max", producer.max); builder.field("min", producer.min); builder.field("sum", producer.sum.value()); builder.field("value_count", producer.count); } - case MetricFieldProducer.AggregateSubMetricFieldProducer producer -> { + case NumericMetricFieldProducer.AggregateSubMetricFieldProducer producer -> { switch (producer.metric) { case max -> builder.field("max", producer.max); case min -> builder.field("min", producer.min); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateSubMetricFieldValueFetcher.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateSubMetricFieldValueFetcher.java index 76b48b834d7e9..e0466409de71a 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateSubMetricFieldValueFetcher.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateSubMetricFieldValueFetcher.java @@ -79,7 +79,7 @@ private AbstractDownsampleFieldProducer createFieldProducer(DownsampleConfig.Sam if (samplingMethod != DownsampleConfig.SamplingMethod.LAST_VALUE) { // If the field is an aggregate_metric_double field, we should use the correct subfields // for each aggregation. This is a downsample-of-downsample case - return new MetricFieldProducer.AggregateSubMetricFieldProducer(aggMetricFieldType.name(), metric); + return new NumericMetricFieldProducer.AggregateSubMetricFieldProducer(aggMetricFieldType.name(), metric); } else { return LastValueFieldProducer.createForAggregateSubMetricMetric(aggMetricFieldType.name(), metric); } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java index 34a55412a0b67..40c546608d68b 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java @@ -367,11 +367,11 @@ public LeafBucketCollector getLeafCollector(final AggregationExecutionContext ag final List nonMetricProducers = new ArrayList<>(); final List formattedDocValues = new ArrayList<>(); - final List metricProducers = new ArrayList<>(); + final List metricProducers = new ArrayList<>(); final List numericDocValues = new ArrayList<>(); for (var fieldValueFetcher : fieldValueFetchers) { var fieldProducer = fieldValueFetcher.fieldProducer(); - if (fieldProducer instanceof MetricFieldProducer metricFieldProducer) { + if (fieldProducer instanceof NumericMetricFieldProducer metricFieldProducer) { metricProducers.add(metricFieldProducer); numericDocValues.add(fieldValueFetcher.getNumericLeaf(ctx)); } else { @@ -385,7 +385,7 @@ public LeafBucketCollector getLeafCollector(final AggregationExecutionContext ag docCountProvider, nonMetricProducers.toArray(new AbstractDownsampleFieldProducer[0]), formattedDocValues.toArray(new FormattedDocValues[0]), - metricProducers.toArray(new MetricFieldProducer[0]), + metricProducers.toArray(new NumericMetricFieldProducer[0]), numericDocValues.toArray(new SortedNumericDoubleValues[0]) ); leafBucketCollectors.add(leafBucketCollector); @@ -407,7 +407,7 @@ class LeafDownsampleCollector extends LeafBucketCollector { final FormattedDocValues[] formattedDocValues; final AbstractDownsampleFieldProducer[] nonMetricProducers; - final MetricFieldProducer[] metricProducers; + final NumericMetricFieldProducer[] metricProducers; final SortedNumericDoubleValues[] numericDocValues; // Capture the first timestamp in order to determine which leaf collector's leafBulkCollection() is invoked first. @@ -420,7 +420,7 @@ class LeafDownsampleCollector extends LeafBucketCollector { DocCountProvider docCountProvider, AbstractDownsampleFieldProducer[] nonMetricProducers, FormattedDocValues[] formattedDocValues, - MetricFieldProducer[] metricProducers, + NumericMetricFieldProducer[] metricProducers, SortedNumericDoubleValues[] numericDocValues ) { assert nonMetricProducers.length == formattedDocValues.length; @@ -506,7 +506,7 @@ void leafBulkCollection() throws IOException { fieldProducer.collect(docValues, docIdBuffer); } for (int i = 0; i < metricProducers.length; i++) { - MetricFieldProducer metricFieldProducer = metricProducers[i]; + NumericMetricFieldProducer metricFieldProducer = metricProducers[i]; SortedNumericDoubleValues numericDoubleValues = numericDocValues[i]; metricFieldProducer.collect(numericDoubleValues, docIdBuffer); } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java index c9913302372bd..01413916485cc 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java @@ -71,7 +71,7 @@ private AbstractDownsampleFieldProducer createFieldProducer(DownsampleConfig.Sam : "Aggregate metric double should be handled by a dedicated FieldValueFetcher"; if (fieldType.getMetricType() != null) { return switch (fieldType.getMetricType()) { - case GAUGE -> MetricFieldProducer.createFieldProducerForGauge(name(), samplingMethod); + case GAUGE -> NumericMetricFieldProducer.createFieldProducerForGauge(name(), samplingMethod); case COUNTER -> LastValueFieldProducer.createForMetric(name()); case HISTOGRAM -> throw new IllegalArgumentException("Unsupported metric type [histogram] for downsampling, coming soon"); // TODO: Support POSITION in downsampling diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/MetricFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldProducer.java similarity index 89% rename from x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/MetricFieldProducer.java rename to x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldProducer.java index 45b594e6587cb..a59ba8a4a719e 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/MetricFieldProducer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/NumericMetricFieldProducer.java @@ -18,20 +18,21 @@ import java.io.IOException; /** - * Class that collects all raw values for a metric field and computes its aggregate (downsampled) + * Class that collects all raw values for a numeric metric field and computes its aggregate (downsampled) * values. Based on the supported metric types, the subclasses of this class compute values for * gauge and metric types. */ -abstract sealed class MetricFieldProducer extends AbstractDownsampleFieldProducer { +abstract sealed class NumericMetricFieldProducer extends AbstractDownsampleFieldProducer { - MetricFieldProducer(String name) { + NumericMetricFieldProducer(String name) { super(name); } @Override public void collect(FormattedDocValues docValues, IntArrayList buffer) throws IOException { - assert false : "MetricFieldProducer does not support formatted doc values"; - throw new UnsupportedOperationException(); + String errorMessage = "MetricFieldProducer does not support formatted doc values"; + assert false : errorMessage; + throw new UnsupportedOperationException(errorMessage); } public abstract void collect(SortedNumericDoubleValues docValues, IntArrayList buffer) throws IOException; @@ -47,9 +48,9 @@ public static AbstractDownsampleFieldProducer createFieldProducerForGauge(String static final double MIN_NO_VALUE = Double.MAX_VALUE; /** - * {@link MetricFieldProducer} implementation for creating an aggregate gauge metric field + * {@link NumericMetricFieldProducer} implementation for creating an aggregate gauge metric field */ - static final class AggregateGaugeMetricFieldProducer extends MetricFieldProducer { + static final class AggregateGaugeMetricFieldProducer extends NumericMetricFieldProducer { double max = MAX_NO_VALUE; double min = MIN_NO_VALUE; @@ -102,7 +103,7 @@ public void write(XContentBuilder builder) throws IOException { } // For downsampling downsampled indices: - static final class AggregateSubMetricFieldProducer extends MetricFieldProducer { + static final class AggregateSubMetricFieldProducer extends NumericMetricFieldProducer { final AggregateMetricDoubleFieldMapper.Metric metric; diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/RestDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/RestDownsampleAction.java index ee8996249ca3b..c2392e7a5e484 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/RestDownsampleAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/RestDownsampleAction.java @@ -30,9 +30,10 @@ @ServerlessScope(Scope.INTERNAL) public class RestDownsampleAction extends BaseRestHandler { - private final Set CAPABILITIES = Stream.of( - "downsample.sampling_mode.last_value", - ).filter(Objects::nonNull).collect(Collectors.toSet()); + private final Set CAPABILITIES = Stream.of("downsample.sampling_mode.last_value") + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + @Override public List routes() { return List.of(new Route(POST, "/{index}/_downsample/{target_index}")); diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldSerializerTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldSerializerTests.java index a6a3b6f5af764..094e43bd775a4 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldSerializerTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldSerializerTests.java @@ -18,13 +18,13 @@ import java.util.List; import static org.elasticsearch.xpack.downsample.LastValueFieldProducerTests.createValuesInstance; -import static org.elasticsearch.xpack.downsample.MetricFieldProducerTests.createNumericValuesInstance; +import static org.elasticsearch.xpack.downsample.NumericMetricFieldProducerTests.createNumericValuesInstance; import static org.hamcrest.Matchers.equalTo; public class AggregateMetricFieldSerializerTests extends ESTestCase { public void testAggregatedGaugeFieldSerialization() throws IOException { - MetricFieldProducer producer = new MetricFieldProducer.AggregateGaugeMetricFieldProducer("my-gauge"); + NumericMetricFieldProducer producer = new NumericMetricFieldProducer.AggregateGaugeMetricFieldProducer("my-gauge"); var docIdBuffer = IntArrayList.from(0, 1, 2); var valuesInstance = createNumericValuesInstance(docIdBuffer, 55.0, 12.2, 5.5); producer.collect(valuesInstance, docIdBuffer); @@ -50,28 +50,28 @@ public void testInvalidCounterFieldSerialization() throws IOException { } public void testAggregatePreAggregatedFieldSerialization() throws IOException { - MetricFieldProducer minProducer = new MetricFieldProducer.AggregateSubMetricFieldProducer( + NumericMetricFieldProducer minProducer = new NumericMetricFieldProducer.AggregateSubMetricFieldProducer( "my-gauge", AggregateMetricDoubleFieldMapper.Metric.min ); var docIdBuffer = IntArrayList.from(0, 1); var valuesInstance = createNumericValuesInstance(docIdBuffer, 10, 5.5); minProducer.collect(valuesInstance, docIdBuffer); - MetricFieldProducer maxProducer = new MetricFieldProducer.AggregateSubMetricFieldProducer( + NumericMetricFieldProducer maxProducer = new NumericMetricFieldProducer.AggregateSubMetricFieldProducer( "my-gauge", AggregateMetricDoubleFieldMapper.Metric.max ); docIdBuffer = IntArrayList.from(0, 1); valuesInstance = createNumericValuesInstance(docIdBuffer, 30, 55.0); maxProducer.collect(valuesInstance, docIdBuffer); - MetricFieldProducer sumProducer = new MetricFieldProducer.AggregateSubMetricFieldProducer( + NumericMetricFieldProducer sumProducer = new NumericMetricFieldProducer.AggregateSubMetricFieldProducer( "my-gauge", AggregateMetricDoubleFieldMapper.Metric.sum ); docIdBuffer = IntArrayList.from(0, 1); valuesInstance = createNumericValuesInstance(docIdBuffer, 30, 72.7); sumProducer.collect(valuesInstance, docIdBuffer); - MetricFieldProducer countProducer = new MetricFieldProducer.AggregateSubMetricFieldProducer( + NumericMetricFieldProducer countProducer = new NumericMetricFieldProducer.AggregateSubMetricFieldProducer( "my-gauge", AggregateMetricDoubleFieldMapper.Metric.value_count ); diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/MetricFieldProducerTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/NumericMetricFieldProducerTests.java similarity index 81% rename from x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/MetricFieldProducerTests.java rename to x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/NumericMetricFieldProducerTests.java index 13453d54215d0..e99df5de59657 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/MetricFieldProducerTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/NumericMetricFieldProducerTests.java @@ -20,11 +20,14 @@ import static org.elasticsearch.xpack.downsample.LastValueFieldProducerTests.createValuesInstance; -public class MetricFieldProducerTests extends AggregatorTestCase { +public class NumericMetricFieldProducerTests extends AggregatorTestCase { public void testMinCountMetric() throws IOException { - var instance = MetricFieldProducer.createFieldProducerForGauge(randomAlphaOfLength(10), DownsampleConfig.SamplingMethod.AGGREGATE); - var aggregateMetricFieldProducer = (MetricFieldProducer.AggregateGaugeMetricFieldProducer) instance; + var instance = NumericMetricFieldProducer.createFieldProducerForGauge( + randomAlphaOfLength(10), + DownsampleConfig.SamplingMethod.AGGREGATE + ); + var aggregateMetricFieldProducer = (NumericMetricFieldProducer.AggregateGaugeMetricFieldProducer) instance; assertEquals(Double.MAX_VALUE, aggregateMetricFieldProducer.min, 0); var docIdBuffer = IntArrayList.from(0, 1, 2, 3); var numericValues = createNumericValuesInstance(docIdBuffer, 40, 5.5, 12.2, 55); @@ -33,7 +36,10 @@ public void testMinCountMetric() throws IOException { aggregateMetricFieldProducer.reset(); assertEquals(Double.MAX_VALUE, aggregateMetricFieldProducer.min, 0); - instance = MetricFieldProducer.createFieldProducerForGauge(randomAlphaOfLength(10), DownsampleConfig.SamplingMethod.LAST_VALUE); + instance = NumericMetricFieldProducer.createFieldProducerForGauge( + randomAlphaOfLength(10), + DownsampleConfig.SamplingMethod.LAST_VALUE + ); var lastValueProducer = (LastValueFieldProducer) instance; assertNull(lastValueProducer.lastValue()); docIdBuffer = IntArrayList.from(0, 1, 2, 3); @@ -45,8 +51,11 @@ public void testMinCountMetric() throws IOException { } public void testMaxCountMetric() throws IOException { - var instance = MetricFieldProducer.createFieldProducerForGauge(randomAlphaOfLength(10), DownsampleConfig.SamplingMethod.AGGREGATE); - var aggregateMetricFieldProducer = (MetricFieldProducer.AggregateGaugeMetricFieldProducer) instance; + var instance = NumericMetricFieldProducer.createFieldProducerForGauge( + randomAlphaOfLength(10), + DownsampleConfig.SamplingMethod.AGGREGATE + ); + var aggregateMetricFieldProducer = (NumericMetricFieldProducer.AggregateGaugeMetricFieldProducer) instance; assertEquals(-Double.MAX_VALUE, aggregateMetricFieldProducer.max, 0); var docIdBuffer = IntArrayList.from(0, 1, 2); var numericValues = createNumericValuesInstance(docIdBuffer, 5.5, 12.2, 55); @@ -55,7 +64,10 @@ public void testMaxCountMetric() throws IOException { aggregateMetricFieldProducer.reset(); assertEquals(-Double.MAX_VALUE, aggregateMetricFieldProducer.max, 0); - instance = MetricFieldProducer.createFieldProducerForGauge(randomAlphaOfLength(10), DownsampleConfig.SamplingMethod.LAST_VALUE); + instance = NumericMetricFieldProducer.createFieldProducerForGauge( + randomAlphaOfLength(10), + DownsampleConfig.SamplingMethod.LAST_VALUE + ); var lastValueProducer = (LastValueFieldProducer) instance; assertNull(lastValueProducer.lastValue()); docIdBuffer = IntArrayList.from(0, 1, 2); @@ -67,8 +79,11 @@ public void testMaxCountMetric() throws IOException { } public void testSumCountMetric() throws IOException { - var instance = MetricFieldProducer.createFieldProducerForGauge(randomAlphaOfLength(10), DownsampleConfig.SamplingMethod.AGGREGATE); - var aggregateMetricFieldProducer = (MetricFieldProducer.AggregateGaugeMetricFieldProducer) instance; + var instance = NumericMetricFieldProducer.createFieldProducerForGauge( + randomAlphaOfLength(10), + DownsampleConfig.SamplingMethod.AGGREGATE + ); + var aggregateMetricFieldProducer = (NumericMetricFieldProducer.AggregateGaugeMetricFieldProducer) instance; assertEquals(0, aggregateMetricFieldProducer.sum.value(), 0); var docIdBuffer = IntArrayList.from(0, 1, 2); var numericValues = createNumericValuesInstance(docIdBuffer, 5.5, 12.2, 55); @@ -77,7 +92,10 @@ public void testSumCountMetric() throws IOException { aggregateMetricFieldProducer.reset(); assertEquals(0, aggregateMetricFieldProducer.sum.value(), 0); - instance = MetricFieldProducer.createFieldProducerForGauge(randomAlphaOfLength(10), DownsampleConfig.SamplingMethod.LAST_VALUE); + instance = NumericMetricFieldProducer.createFieldProducerForGauge( + randomAlphaOfLength(10), + DownsampleConfig.SamplingMethod.LAST_VALUE + ); var lastValueProducer = (LastValueFieldProducer) instance; assertNull(lastValueProducer.lastValue()); docIdBuffer = IntArrayList.from(0, 1, 2); @@ -93,7 +111,7 @@ public void testSumCountMetric() throws IOException { * Tests stolen from SumAggregatorTests#testSummationAccuracy */ public void testSummationAccuracy() throws IOException { - var instance = new MetricFieldProducer.AggregateGaugeMetricFieldProducer(randomAlphaOfLength(10)); + var instance = new NumericMetricFieldProducer.AggregateGaugeMetricFieldProducer(randomAlphaOfLength(10)); assertEquals(0, instance.sum.value(), 0); var docIdBuffer = IntArrayList.from(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); // Summing up a normal array and expect an accurate value @@ -165,8 +183,11 @@ public void testSummationAccuracy() throws IOException { } public void testValueCountMetric() throws IOException { - var instance = MetricFieldProducer.createFieldProducerForGauge(randomAlphaOfLength(10), DownsampleConfig.SamplingMethod.AGGREGATE); - var aggregateMetricFieldProducer = (MetricFieldProducer.AggregateGaugeMetricFieldProducer) instance; + var instance = NumericMetricFieldProducer.createFieldProducerForGauge( + randomAlphaOfLength(10), + DownsampleConfig.SamplingMethod.AGGREGATE + ); + var aggregateMetricFieldProducer = (NumericMetricFieldProducer.AggregateGaugeMetricFieldProducer) instance; assertEquals(0, aggregateMetricFieldProducer.count); var docIdBuffer = IntArrayList.from(0, 1, 2); var numericValues = createNumericValuesInstance(docIdBuffer, 40, 30, 20); @@ -175,7 +196,10 @@ public void testValueCountMetric() throws IOException { instance.reset(); assertEquals(0, aggregateMetricFieldProducer.count); - instance = MetricFieldProducer.createFieldProducerForGauge(randomAlphaOfLength(10), DownsampleConfig.SamplingMethod.LAST_VALUE); + instance = NumericMetricFieldProducer.createFieldProducerForGauge( + randomAlphaOfLength(10), + DownsampleConfig.SamplingMethod.LAST_VALUE + ); var lastValueProducer = (LastValueFieldProducer) instance; assertNull(lastValueProducer.lastValue()); docIdBuffer = IntArrayList.from(0, 1, 2); @@ -207,7 +231,7 @@ public void testCounterMetricFieldProducer() throws IOException { public void testGaugeMetricFieldProducer() throws IOException { final String field = "field"; - MetricFieldProducer producer = new MetricFieldProducer.AggregateGaugeMetricFieldProducer(field); + NumericMetricFieldProducer producer = new NumericMetricFieldProducer.AggregateGaugeMetricFieldProducer(field); assertTrue(producer.isEmpty()); var docIdBuffer = IntArrayList.from(0, 1, 2); var valuesInstance = createNumericValuesInstance(docIdBuffer, 55.0, 12.2, 5.5); From 59953e0aad858e20e59ab1df9adc321583648f3a Mon Sep 17 00:00:00 2001 From: gmarouli Date: Tue, 25 Nov 2025 12:13:51 +0200 Subject: [PATCH 3/8] Create exponential histogram field producer for downsampling --- .../ExponentialHistogramMerger.java | 11 ++- ...ponentialHistogramMetricFieldProducer.java | 84 +++++++++++++++++++ .../downsample/LastValueFieldProducer.java | 19 +++++ .../ExponentialHistogramFieldMapper.java | 2 +- 4 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramMetricFieldProducer.java diff --git a/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMerger.java b/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMerger.java index 60ad6f8d8d197..77be4a0d4d244 100644 --- a/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMerger.java +++ b/libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMerger.java @@ -36,7 +36,8 @@ * while keeping the bucket count in the result below a given limit. */ public class ExponentialHistogramMerger implements Accountable, Releasable { - + // OpenTelemetry SDK default, we might make this configurable later + private static final int MAX_HISTOGRAM_BUCKETS = 320; private static final long BASE_SIZE = RamUsageEstimator.shallowSizeOfInstance(ExponentialHistogramMerger.class) + DownscaleStats.SIZE; // Our algorithm is not in-place, therefore we use two histograms and ping-pong between them @@ -53,6 +54,14 @@ public class ExponentialHistogramMerger implements Accountable, Releasable { private final ExponentialHistogramCircuitBreaker circuitBreaker; private boolean closed = false; + /** + * Creates a new instance with the OpenTelemetry SDK default bucket limit of {@link ExponentialHistogramMerger#MAX_HISTOGRAM_BUCKETS} + * @param circuitBreaker the circuit breaker to use to limit memory allocations + */ + public static ExponentialHistogramMerger create(ExponentialHistogramCircuitBreaker circuitBreaker) { + return create(MAX_HISTOGRAM_BUCKETS, circuitBreaker); + } + /** * Creates a new instance with the specified bucket limit. * diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramMetricFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramMetricFieldProducer.java new file mode 100644 index 0000000000000..0dd53b621f4ce --- /dev/null +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramMetricFieldProducer.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.downsample; + +import org.apache.lucene.internal.hppc.IntArrayList; +import org.elasticsearch.action.downsample.DownsampleConfig; +import org.elasticsearch.exponentialhistogram.ExponentialHistogram; +import org.elasticsearch.exponentialhistogram.ExponentialHistogramCircuitBreaker; +import org.elasticsearch.exponentialhistogram.ExponentialHistogramMerger; +import org.elasticsearch.exponentialhistogram.ExponentialHistogramXContent; +import org.elasticsearch.index.fielddata.FormattedDocValues; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.exponentialhistogram.fielddata.ExponentialHistogramValuesReader; + +import java.io.IOException; + +/** + * Class that collects all raw values for an exponential histogram metric field and computes its aggregate (downsampled) + * values. Based on the supported metric types, the subclasses of this class compute values for + * gauge and metric types. + */ +final class ExponentialHistogramMetricFieldProducer extends AbstractDownsampleFieldProducer { + + private ExponentialHistogramMerger merger = null; + + ExponentialHistogramMetricFieldProducer(String name) { + super(name); + } + + /** + * @return the requested produces based on the sampling method for metric of type exponential histogram + */ + public static AbstractDownsampleFieldProducer createMetricProducerForExponentialHistogram( + String name, + DownsampleConfig.SamplingMethod samplingMethod + ) { + return switch (samplingMethod) { + case AGGREGATE -> new ExponentialHistogramMetricFieldProducer(name); + case LAST_VALUE -> new LastValueFieldProducer.ExponentialHistogramFieldProducer(name); + }; + } + + @Override + public void collect(FormattedDocValues docValues, IntArrayList buffer) throws IOException { + String errorMessage = "MetricFieldProducer does not support formatted doc values"; + assert false : errorMessage; + throw new UnsupportedOperationException(errorMessage); + } + + public void collect(ExponentialHistogramValuesReader docValues, IntArrayList docIdBuffer) throws IOException { + for (int i = 0; i < docIdBuffer.size(); i++) { + int docId = docIdBuffer.get(i); + if (docValues.advanceExact(docId) == false) { + continue; + } + isEmpty = false; + if (merger == null) { + merger = ExponentialHistogramMerger.create(ExponentialHistogramCircuitBreaker.noop()); + } + ExponentialHistogram value = docValues.histogramValue(); + merger.add(value); + } + } + + @Override + public void reset() { + isEmpty = true; + merger = null; + } + + @Override + public void write(XContentBuilder builder) throws IOException { + if (isEmpty() == false) { + builder.field(name()); + ExponentialHistogramXContent.serialize(builder, merger.get()); + merger.close(); + } + } +} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldProducer.java index e6a1766c28fa6..794970f02b18d 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldProducer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/LastValueFieldProducer.java @@ -9,6 +9,8 @@ import org.apache.lucene.internal.hppc.IntArrayList; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.exponentialhistogram.ExponentialHistogram; +import org.elasticsearch.exponentialhistogram.ExponentialHistogramXContent; import org.elasticsearch.index.fielddata.FormattedDocValues; import org.elasticsearch.index.fielddata.HistogramValue; import org.elasticsearch.index.mapper.flattened.FlattenedFieldSyntheticWriterHelper; @@ -44,6 +46,8 @@ static LastValueFieldProducer createForLabel(String name, String fieldType) { : "field type cannot be aggregate metric double: " + fieldType + " for field " + name; if ("histogram".equals(fieldType)) { return new LastValueFieldProducer.HistogramFieldProducer(name, true); + } else if ("exponential_histogram".equals(fieldType)) { + return new LastValueFieldProducer.ExponentialHistogramFieldProducer(name); } else if ("flattened".equals(fieldType)) { return new LastValueFieldProducer.FlattenedFieldProducer(name, true); } @@ -199,4 +203,19 @@ public void write(XContentBuilder builder) throws IOException { } } } + + static class ExponentialHistogramFieldProducer extends LastValueFieldProducer { + ExponentialHistogramFieldProducer(String name) { + // Exponential histograms do not support multi value anyway + super(name, false); + } + + @Override + public void write(XContentBuilder builder) throws IOException { + if (isEmpty() == false) { + builder.field(name()); + ExponentialHistogramXContent.serialize(builder, (ExponentialHistogram) lastValue()); + } + } + } } diff --git a/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java b/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java index 471b418a06acf..ed3626a95760b 100644 --- a/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java +++ b/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java @@ -288,7 +288,7 @@ public ExponentialHistogramValuesReader getHistogramValues() throws IOException @Override public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { - throw new UnsupportedOperationException("The [" + CONTENT_TYPE + "] field does not " + "support scripts"); + throw new UnsupportedOperationException("The [" + CONTENT_TYPE + "] field does not support scripts"); } @Override From 84940843f68affbeb5f9014922fde05d1e5ddc5a Mon Sep 17 00:00:00 2001 From: gmarouli Date: Tue, 25 Nov 2025 12:14:28 +0200 Subject: [PATCH 4/8] Downsample exponential histograms --- .../downsample/DownsampleShardIndexer.java | 38 +++++++++++++++---- .../xpack/downsample/FieldValueFetcher.java | 13 ++++++- .../downsample/RestDownsampleAction.java | 10 +++-- .../downsample/TransportDownsampleAction.java | 15 ++++---- 4 files changed, 57 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java index 40c546608d68b..adc452846e857 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java @@ -58,6 +58,7 @@ import org.elasticsearch.xpack.core.downsample.DownsampleShardIndexerStatus; import org.elasticsearch.xpack.core.downsample.DownsampleShardPersistentTaskState; import org.elasticsearch.xpack.core.downsample.DownsampleShardTask; +import org.elasticsearch.xpack.core.exponentialhistogram.fielddata.ExponentialHistogramValuesReader; import java.io.Closeable; import java.io.IOException; @@ -364,18 +365,25 @@ public LeafBucketCollector getLeafCollector(final AggregationExecutionContext ag docCountProvider.setLeafReaderContext(ctx); // For each field, return a tuple with the downsample field producer and the field value leaf - final List nonMetricProducers = new ArrayList<>(); + final List lastValueProducers = new ArrayList<>(); final List formattedDocValues = new ArrayList<>(); - final List metricProducers = new ArrayList<>(); + final List numericValueProducers = new ArrayList<>(); final List numericDocValues = new ArrayList<>(); + + final List exponentialHistogramProducers = new ArrayList<>(); + final List exponentialHistogramDocValues = new ArrayList<>(); + for (var fieldValueFetcher : fieldValueFetchers) { var fieldProducer = fieldValueFetcher.fieldProducer(); if (fieldProducer instanceof NumericMetricFieldProducer metricFieldProducer) { - metricProducers.add(metricFieldProducer); + numericValueProducers.add(metricFieldProducer); numericDocValues.add(fieldValueFetcher.getNumericLeaf(ctx)); + } else if (fieldProducer instanceof ExponentialHistogramMetricFieldProducer exponentialHistogramProducer) { + exponentialHistogramProducers.add(exponentialHistogramProducer); + exponentialHistogramDocValues.add(fieldValueFetcher.getExponentialHistogramLeaf(ctx)); } else { - nonMetricProducers.add(fieldProducer); + lastValueProducers.add(fieldProducer); formattedDocValues.add(fieldValueFetcher.getLeaf(ctx)); } } @@ -383,10 +391,12 @@ public LeafBucketCollector getLeafCollector(final AggregationExecutionContext ag var leafBucketCollector = new LeafDownsampleCollector( aggCtx, docCountProvider, - nonMetricProducers.toArray(new AbstractDownsampleFieldProducer[0]), + lastValueProducers.toArray(new AbstractDownsampleFieldProducer[0]), formattedDocValues.toArray(new FormattedDocValues[0]), - metricProducers.toArray(new NumericMetricFieldProducer[0]), - numericDocValues.toArray(new SortedNumericDoubleValues[0]) + numericValueProducers.toArray(new NumericMetricFieldProducer[0]), + numericDocValues.toArray(new SortedNumericDoubleValues[0]), + exponentialHistogramProducers.toArray(new ExponentialHistogramMetricFieldProducer[0]), + exponentialHistogramDocValues.toArray(new ExponentialHistogramValuesReader[0]) ); leafBucketCollectors.add(leafBucketCollector); return leafBucketCollector; @@ -409,6 +419,8 @@ class LeafDownsampleCollector extends LeafBucketCollector { final NumericMetricFieldProducer[] metricProducers; final SortedNumericDoubleValues[] numericDocValues; + final ExponentialHistogramMetricFieldProducer[] experimentalHistogramProducers; + final ExponentialHistogramValuesReader[] exponentialHistogramDocValues; // Capture the first timestamp in order to determine which leaf collector's leafBulkCollection() is invoked first. long firstTimeStampForBulkCollection; @@ -421,10 +433,13 @@ class LeafDownsampleCollector extends LeafBucketCollector { AbstractDownsampleFieldProducer[] nonMetricProducers, FormattedDocValues[] formattedDocValues, NumericMetricFieldProducer[] metricProducers, - SortedNumericDoubleValues[] numericDocValues + SortedNumericDoubleValues[] numericDocValues, + ExponentialHistogramMetricFieldProducer[] experimentalHistogramProducers, + ExponentialHistogramValuesReader[] exponentialHistogramDocValues ) { assert nonMetricProducers.length == formattedDocValues.length; assert metricProducers.length == numericDocValues.length; + assert experimentalHistogramProducers.length == exponentialHistogramDocValues.length; this.aggCtx = aggCtx; this.docCountProvider = docCountProvider; @@ -432,6 +447,8 @@ class LeafDownsampleCollector extends LeafBucketCollector { this.formattedDocValues = formattedDocValues; this.metricProducers = metricProducers; this.numericDocValues = numericDocValues; + this.experimentalHistogramProducers = experimentalHistogramProducers; + this.exponentialHistogramDocValues = exponentialHistogramDocValues; } @Override @@ -510,6 +527,11 @@ void leafBulkCollection() throws IOException { SortedNumericDoubleValues numericDoubleValues = numericDocValues[i]; metricFieldProducer.collect(numericDoubleValues, docIdBuffer); } + for (int i = 0; i < experimentalHistogramProducers.length; i++) { + var experimentalHistogramProducer = experimentalHistogramProducers[i]; + ExponentialHistogramValuesReader exponentialHistogramValuesReader = exponentialHistogramDocValues[i]; + experimentalHistogramProducer.collect(exponentialHistogramValuesReader, docIdBuffer); + } docsProcessed += docIdBuffer.size(); task.setDocsProcessed(docsProcessed); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java index 01413916485cc..8691976f62531 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java @@ -18,7 +18,10 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper; +import org.elasticsearch.xpack.core.exponentialhistogram.fielddata.ExponentialHistogramValuesReader; +import org.elasticsearch.xpack.core.exponentialhistogram.fielddata.LeafExponentialHistogramFieldData; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -62,6 +65,11 @@ public SortedNumericDoubleValues getNumericLeaf(LeafReaderContext context) { return numericFieldData.getDoubleValues(); } + public ExponentialHistogramValuesReader getExponentialHistogramLeaf(LeafReaderContext context) throws IOException { + LeafExponentialHistogramFieldData exponentialHistogramFieldData = (LeafExponentialHistogramFieldData) fieldData.load(context); + return exponentialHistogramFieldData.getHistogramValues(); + } + public AbstractDownsampleFieldProducer fieldProducer() { return fieldProducer; } @@ -73,7 +81,10 @@ private AbstractDownsampleFieldProducer createFieldProducer(DownsampleConfig.Sam return switch (fieldType.getMetricType()) { case GAUGE -> NumericMetricFieldProducer.createFieldProducerForGauge(name(), samplingMethod); case COUNTER -> LastValueFieldProducer.createForMetric(name()); - case HISTOGRAM -> throw new IllegalArgumentException("Unsupported metric type [histogram] for downsampling, coming soon"); + case HISTOGRAM -> ExponentialHistogramMetricFieldProducer.createMetricProducerForExponentialHistogram( + name(), + samplingMethod + ); // TODO: Support POSITION in downsampling case POSITION -> throw new IllegalArgumentException("Unsupported metric type [position] for down-sampling"); }; diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/RestDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/RestDownsampleAction.java index c2392e7a5e484..9ffb0b39a3b90 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/RestDownsampleAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/RestDownsampleAction.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.downsample.DownsampleAction; import org.elasticsearch.action.downsample.DownsampleConfig; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.core.TimeValue; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -30,9 +31,12 @@ @ServerlessScope(Scope.INTERNAL) public class RestDownsampleAction extends BaseRestHandler { - private final Set CAPABILITIES = Stream.of("downsample.sampling_mode.last_value") - .filter(Objects::nonNull) - .collect(Collectors.toSet()); + public static final FeatureFlag EXPONENTIAL_HISTOGRAM_FEATURE = new FeatureFlag("exponential_histogram"); + + private final Set CAPABILITIES = Stream.of( + "downsample.sampling_mode.last_value", + EXPONENTIAL_HISTOGRAM_FEATURE.isEnabled() ? "downsampling.exponential_histograms" : null + ).filter(Objects::nonNull).collect(Collectors.toSet()); @Override public List routes() { diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index ea6f33c903009..344b030abaa75 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -839,19 +839,20 @@ private static void addMetricFieldMapping(final XContentBuilder builder, final S fieldProperties.get(TIME_SERIES_METRIC_PARAM).toString() ); builder.startObject(field); - if (metricType == TimeSeriesParams.MetricType.COUNTER) { - // For counters, we keep the same field type, because they store - // only one value (the last value of the counter) - for (String fieldProperty : fieldProperties.keySet()) { - builder.field(fieldProperty, fieldProperties.get(fieldProperty)); - } - } else { + if (metricType == TimeSeriesParams.MetricType.GAUGE) { var supported = getSupportedMetrics(metricType, fieldProperties); builder.field("type", AggregateMetricDoubleFieldMapper.CONTENT_TYPE) .stringListField(AggregateMetricDoubleFieldMapper.Names.METRICS, supported.supportedMetrics) .field(AggregateMetricDoubleFieldMapper.Names.DEFAULT_METRIC, supported.defaultMetric) .field(TIME_SERIES_METRIC_PARAM, metricType); + } else { + // For counters and histograms, we keep the same field type. + // Counters because they store the last value (for now) + // Histograms because they are merged to a histogram + for (String fieldProperty : fieldProperties.keySet()) { + builder.field(fieldProperty, fieldProperties.get(fieldProperty)); + } } builder.endObject(); } From 9f08780fdb8e167da00d150b63450ffa3e7d9625 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Tue, 25 Nov 2025 12:14:34 +0200 Subject: [PATCH 5/8] Add tests --- x-pack/plugin/downsample/build.gradle | 1 + x-pack/plugin/downsample/qa/rest/build.gradle | 1 + .../xpack/downsample/DownsampleRestIT.java | 1 + .../test/downsample/10_basic.yml | 165 ++++++++++++++++++ .../xpack/downsample/DownsampleIT.java | 54 +++++- .../downsample/DownsamplingIntegTestCase.java | 5 +- 6 files changed, 224 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/downsample/build.gradle b/x-pack/plugin/downsample/build.gradle index 821052055170e..99a16de2966cc 100644 --- a/x-pack/plugin/downsample/build.gradle +++ b/x-pack/plugin/downsample/build.gradle @@ -17,6 +17,7 @@ dependencies { testImplementation project(xpackModule('ilm')) compileOnly project(xpackModule('analytics')) compileOnly project(xpackModule('mapper-aggregate-metric')) + compileOnly project(xpackModule('mapper-exponential-histogram')) testImplementation(testArtifact(project(xpackModule('core')))) testImplementation project(xpackModule('ccr')) testImplementation project(xpackModule('esql')) diff --git a/x-pack/plugin/downsample/qa/rest/build.gradle b/x-pack/plugin/downsample/qa/rest/build.gradle index c59ae5287ce11..e3f456495d3ef 100644 --- a/x-pack/plugin/downsample/qa/rest/build.gradle +++ b/x-pack/plugin/downsample/qa/rest/build.gradle @@ -21,6 +21,7 @@ dependencies { clusterModules project(':modules:data-streams') clusterModules project(':modules:aggregations') clusterModules project(':modules:ingest-common') + clusterModules project(xpackModule('mapper-exponential-histogram')) } restResources { diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java index d6ffa14bfab2b..0f4c205a9a4d9 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java @@ -25,6 +25,7 @@ public class DownsampleRestIT extends ESClientYamlSuiteTestCase { .module("mapper-extras") // for scaled_float .module("x-pack-analytics") // for histogram .module("data-streams") // for time series + .module("exponential-histogram") .module("ingest-common") .setting("xpack.license.self_generated.type", "trial") .setting("xpack.security.enabled", "false") diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml index 74e572708cbab..4e76a71011fa9 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml @@ -1846,3 +1846,168 @@ setup: - match: { hits.hits.1._source._doc_count: 1 } - match: { hits.hits.1._source.k8s\.pod\.name: cat } - match: { hits.hits.1._source.k8s\.pod\.empty: "" } + +--- +"Downsample exponential histogram with last value": + - requires: + capabilities: + - method: POST + path: /{index}/_downsample/{target_index} + capabilities: [ "downsample.sampling_mode.last_value", "downsampling.exponential_histograms" ] + test_runner_features: [ "capabilities" ] + reason: Last value sampling method was added in 9.3 + - do: + indices.create: + index: test-exponential-histogram + body: + settings: + number_of_shards: 1 + index: + mode: time_series + routing_path: [ metricset ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + metric: + type: exponential_histogram + time_series_metric: histogram + label: + type: exponential_histogram + - is_true: shards_acknowledged + + - do: + bulk: + refresh: true + index: test-exponential-histogram + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "lala", "metric": {"scale": -1, "sum": 1.2295711150758473, "min": -0.9714203537912545, "max": 0.9865998839317596, "negative": {"indices": [-3, -2, -1], "counts": [2, 11, 36]}, "positive": {"indices": [-6, -3, -2, -1], "counts": [1, 3, 11, 36]}}, "label": {"scale": -1, "sum": 1.2295711150758473, "min": -0.9714203537912545, "max": 0.9865998839317596, "negative": {"indices": [-3, -2, -1], "counts": [2, 11, 36]}, "positive": {"indices": [-6, -3, -2, -1], "counts": [1, 3, 11, 36]}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "lala", "metric": {"scale": 1, "sum": 5.89290987609197, "min": -0.9410989907817573, "max": 0.9821606962370899, "negative": {"indices": [-9, -8, -7, -4, -3, -2, -1], "counts": [1, 2, 1, 3, 3, 2, 6]}, "positive": {"indices": [-10, -7, -6, -5, -4, -3, -2, -1], "counts": [3, 2, 1, 2, 5, 6, 4, 9]}}, "label": {"scale": 1, "sum": 5.89290987609197, "min": -0.9410989907817573, "max": 0.9821606962370899, "negative": {"indices": [-9, -8, -7, -4, -3, -2, -1], "counts": [1, 2, 1, 3, 3, 2, 6]}, "positive": {"indices": [-10, -7, -6, -5, -4, -3, -2, -1], "counts": [3, 2, 1, 2, 5, 6, 4, 9]}}}' + - match: { items.0.index.result: "created"} + - match: { items.1.index.result: "created"} + + - do: + indices.put_settings: + index: test-exponential-histogram + body: + index.blocks.write: true + - is_true: acknowledged + + - do: + indices.downsample: + index: test-exponential-histogram + target_index: downsampled-exponential-histogram + body: > + { + "fixed_interval": "1h", + "sampling_method": "last_value" + } + - is_true: acknowledged + + - do: + search: + index: downsampled-exponential-histogram + + - length: { hits.hits: 1 } + + - match: { hits.hits.0._source._doc_count: 2 } + - match: { hits.hits.0._source.metricset: lala } + - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } + - match: { hits.hits.0._source.metric.max: 0.9821606962370899 } + - match: { hits.hits.0._source.metric.min: -0.9410989907817573 } + - match: { hits.hits.0._source.metric.scale: 1 } + - match: { hits.hits.0._source.metric.sum: 5.89290987609197 } + - match: { hits.hits.0._source.label.max: 0.9821606962370899 } + - match: { hits.hits.0._source.label.min: -0.9410989907817573 } + - match: { hits.hits.0._source.label.scale: 1 } + - match: { hits.hits.0._source.label.sum: 5.89290987609197 } + +--- +"Downsample exponential histogram with aggregate": + - requires: + capabilities: + - method: POST + path: /{index}/_downsample/{target_index} + capabilities: [ "downsampling.exponential_histograms" ] + test_runner_features: [ "capabilities" ] + reason: Last value sampling method was added in 9.3 + - do: + indices.create: + index: test-exponential-histogram + body: + settings: + number_of_shards: 1 + index: + mode: time_series + routing_path: [ metricset ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + metric: + type: exponential_histogram + time_series_metric: histogram + label: + type: exponential_histogram + - is_true: shards_acknowledged + + - do: + bulk: + refresh: true + index: test-exponential-histogram + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "lala", "metric": {"scale":38,"sum":-2.659439830689209,"min":-0.9985699620066124,"max":0.9991606633362218,"positive":{"indices":[-1349015837690,-1133741949749,-1093555204471,-878875428260,-870694284095,-831557211207,-801793317707,-633655864830,-583473541038,-576512037819,-568470393517,-555322469609,-510705457264,-497886869173,-485920366989,-459535642468,-433595368145,-390027945362,-356113581539,-343989550188,-340618512768,-292750567759,-278017998597,-273632142061,-265328573027,-262313863416,-261799809534,-243544433917,-200039973732,-189492605429,-153154647965,-134459061772,-117996065668,-111711978805,-109965984611,-101785856856,-93187164407,-79681306932,-76611651392,-65149069301,-58423697696,-56765280861,-41035363279,-39729265577,-7007278892,-332991304],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1722041245251,-1507876068779,-1461993774485,-1437177146905,-1022221748846,-909329959788,-902070008615,-869527704039,-859349507080,-836401169293,-835541219174,-827446708753,-823853818576,-767042645194,-764466131268,-698239247988,-598695656860,-560446067886,-554301612745,-521812328033,-508700015492,-462218888037,-417010270052,-384781248780,-371405487026,-366789447120,-349996665670,-300317310192,-287581746174,-287132961026,-284865058959,-255687331935,-236975923503,-200065874687,-196100184062,-147243768507,-145933242978,-140904484007,-136697299301,-117022220796,-87401418251,-63863967756,-62902280835,-59450473434,-53232718483,-49452610935,-45384224466,-36361788727,-34176300034,-12469429575,-9431189993,-3254481828,-2083495024,-567508884],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}, "label": {"scale":38,"sum":-2.659439830689209,"min":-0.9985699620066124,"max":0.9991606633362218,"positive":{"indices":[-1349015837690,-1133741949749,-1093555204471,-878875428260,-870694284095,-831557211207,-801793317707,-633655864830,-583473541038,-576512037819,-568470393517,-555322469609,-510705457264,-497886869173,-485920366989,-459535642468,-433595368145,-390027945362,-356113581539,-343989550188,-340618512768,-292750567759,-278017998597,-273632142061,-265328573027,-262313863416,-261799809534,-243544433917,-200039973732,-189492605429,-153154647965,-134459061772,-117996065668,-111711978805,-109965984611,-101785856856,-93187164407,-79681306932,-76611651392,-65149069301,-58423697696,-56765280861,-41035363279,-39729265577,-7007278892,-332991304],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1722041245251,-1507876068779,-1461993774485,-1437177146905,-1022221748846,-909329959788,-902070008615,-869527704039,-859349507080,-836401169293,-835541219174,-827446708753,-823853818576,-767042645194,-764466131268,-698239247988,-598695656860,-560446067886,-554301612745,-521812328033,-508700015492,-462218888037,-417010270052,-384781248780,-371405487026,-366789447120,-349996665670,-300317310192,-287581746174,-287132961026,-284865058959,-255687331935,-236975923503,-200065874687,-196100184062,-147243768507,-145933242978,-140904484007,-136697299301,-117022220796,-87401418251,-63863967756,-62902280835,-59450473434,-53232718483,-49452610935,-45384224466,-36361788727,-34176300034,-12469429575,-9431189993,-3254481828,-2083495024,-567508884],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "lala", "metric": {"scale":38,"sum":-2.612341035444145,"min":-0.8596842630789088,"max":0.9412120998532649,"positive":{"indices":[-1527782821719,-1491647186305,-912103649399,-744031889051,-669036239523,-602111557612,-586601121715,-567461548128,-483021101785,-450865160617,-331097834736,-293981770197,-289290353357,-246965149109,-203070759894,-180702903638,-135653049938,-93843302593,-78356307212,-76729181967,-65070806329,-64241350555,-24026590621],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1423777207002,-923870284341,-877992064499,-870206177266,-850685295282,-759799372292,-643924938249,-570457241089,-528116212727,-369003212885,-315079526687,-292745353109,-235218433328,-222024648135,-204603508425,-195434414609,-177993661419,-172367743205,-168351131368,-149696012047,-115115447928,-95206374327,-74115574931,-74013160315,-64191774880,-63955626713,-59956698242],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}, "label": {"scale":38,"sum":-2.612341035444145,"min":-0.8596842630789088,"max":0.9412120998532649,"positive":{"indices":[-1527782821719,-1491647186305,-912103649399,-744031889051,-669036239523,-602111557612,-586601121715,-567461548128,-483021101785,-450865160617,-331097834736,-293981770197,-289290353357,-246965149109,-203070759894,-180702903638,-135653049938,-93843302593,-78356307212,-76729181967,-65070806329,-64241350555,-24026590621],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},"negative":{"indices":[-1423777207002,-923870284341,-877992064499,-870206177266,-850685295282,-759799372292,-643924938249,-570457241089,-528116212727,-369003212885,-315079526687,-292745353109,-235218433328,-222024648135,-204603508425,-195434414609,-177993661419,-172367743205,-168351131368,-149696012047,-115115447928,-95206374327,-74115574931,-74013160315,-64191774880,-63955626713,-59956698242],"counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}}}' + - match: { items.0.index.result: "created"} + - match: { items.1.index.result: "created"} + + - do: + indices.put_settings: + index: test-exponential-histogram + body: + index.blocks.write: true + - is_true: acknowledged + + - do: + indices.downsample: + index: test-exponential-histogram + target_index: downsampled-exponential-histogram-aggregate + body: > + { + "fixed_interval": "1h" + } + - is_true: acknowledged + + - do: + search: + index: downsampled-exponential-histogram-aggregate + + - length: { hits.hits: 1 } + + - match: { hits.hits.0._source._doc_count: 2 } + - match: { hits.hits.0._source.metricset: lala } + - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } + - match: { hits.hits.0._source.metric.max: 0.9991606633362218 } + - match: { hits.hits.0._source.metric.min: -0.9985699620066124 } + - match: { hits.hits.0._source.metric.scale: 38 } + - match: { hits.hits.0._source.metric.sum: -5.271780866133354 } + - match: { hits.hits.0._source.label.max: 0.9412120998532649 } + - match: { hits.hits.0._source.label.min: -0.8596842630789088 } + - match: { hits.hits.0._source.label.scale: 38 } + - match: { hits.hits.0._source.label.sum: -2.612341035444145 } diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleIT.java index 8e2bce4251251..3d429bbacbfca 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleIT.java @@ -128,7 +128,11 @@ public void testDownsamplingPassthroughMetrics() throws Exception { downsampleAndAssert(dataStreamName, mapping, sourceSupplier, randomSamplingMethod()); } - public void testLastValueMode() throws Exception { + public void testLastValueMethod() throws Exception { + downsampleWithSamplingMethod(DownsampleConfig.SamplingMethod.LAST_VALUE); + } + + private void downsampleWithSamplingMethod(DownsampleConfig.SamplingMethod method) throws Exception { String dataStreamName = "metrics-foo"; String mapping = """ { @@ -148,6 +152,10 @@ public void testLastValueMode() throws Exception { "type": "double", "time_series_metric": "gauge" }, + "metrics.latency": { + "type": "exponential_histogram", + "time_series_metric": "histogram" + }, "my_labels": { "properties": { "my_histogram": { @@ -157,6 +165,9 @@ public void testLastValueMode() throws Exception { "type": "aggregate_metric_double", "metrics": [ "min", "max", "sum", "value_count" ], "default_metric": "max" + }, + "my_exponential_histogram": { + "type": "exponential_histogram" } } } @@ -177,6 +188,25 @@ public void testLastValueMode() throws Exception { .field("attributes.os.name", randomFrom("linux", "windows", "macos")) .field("metrics.cpu_usage", randomDouble()) + .startObject("metrics.latency") + .field("scale", 0) + .field("sum", -3775.0) + .field("min", -100.0) + .field("max", 50.0) + .startObject("zero") + .field("count", 1) + .field("threshold", 1.0E-4) + .endObject() + .startObject("positive") + .array("indices", new int[] { -1, 0, 1, 2, 3, 4, 5 }) + .array("counts", new int[] { 1, 1, 2, 4, 8, 16, 18 }) + .endObject() + .startObject("negative") + .array("indices", new int[] { -1, 0, 1, 2, 3, 4, 5, 6 }) + .array("counts", new int[] { 1, 1, 2, 4, 8, 16, 32, 36 }) + .endObject() + .endObject() + .startObject("my_labels.my_histogram") .array("values", randomHistogramValues(maxHistogramSize)) .array("counts", randomHistogramValueCounts(maxHistogramSize)) @@ -188,12 +218,32 @@ public void testLastValueMode() throws Exception { .field("sum", randomFloatBetween(20.0f, 30.0f, true)) .field("value_count", randomIntBetween(1, 10)) .endObject() + + .startObject("my_labels.my_exponential_histogram") + .field("scale", 0) + .field("sum", -3775.0) + .field("min", -100.0) + .field("max", 50.0) + .startObject("zero") + .field("count", 1) + .field("threshold", 1.0E-4) + .endObject() + .startObject("positive") + .array("indices", new int[] { -1, 0, 1, 2, 3, 4, 5 }) + .array("counts", new int[] { 1, 1, 2, 4, 8, 16, 18 }) + .endObject() + .startObject("negative") + .array("indices", new int[] { -1, 0, 1, 2, 3, 4, 5, 6 }) + .array("counts", new int[] { 1, 1, 2, 4, 8, 16, 32, 36 }) + .endObject() + .endObject() + .endObject(); } catch (IOException e) { throw new RuntimeException(e); } }; - downsampleAndAssert(dataStreamName, mapping, sourceSupplier, DownsampleConfig.SamplingMethod.LAST_VALUE); + downsampleAndAssert(dataStreamName, mapping, sourceSupplier, method); } /** diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsamplingIntegTestCase.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsamplingIntegTestCase.java index ab91aabbdeec1..4ed9ab5a58164 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsamplingIntegTestCase.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsamplingIntegTestCase.java @@ -47,6 +47,7 @@ import org.elasticsearch.xpack.analytics.AnalyticsPlugin; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; +import org.elasticsearch.xpack.exponentialhistogram.ExponentialHistogramMapperPlugin; import java.io.IOException; import java.time.LocalDateTime; @@ -90,7 +91,8 @@ protected Collection> nodePlugins() { Downsample.class, AggregateMetricMapperPlugin.class, EsqlPlugin.class, - AnalyticsPlugin.class + AnalyticsPlugin.class, + ExponentialHistogramMapperPlugin.class ); } @@ -302,6 +304,7 @@ void assertDownsampleIndexFieldsAndDimensions(String sourceIndex, String downsam assertThat(fieldMapping.get("type"), equalTo("aggregate_metric_double")); } } + case HISTOGRAM -> assertThat(fieldMapping.get("type"), equalTo("exponential_histogram")); default -> fail("Unsupported field type"); } assertThat(fieldMapping.get("time_series_metric"), equalTo(metricType.toString())); From 8af93c4e922a4b4edeae2bc4141974c09cea5b21 Mon Sep 17 00:00:00 2001 From: gmarouli Date: Tue, 25 Nov 2025 13:32:50 +0200 Subject: [PATCH 6/8] Add exponential histogram mapper to all rest tests --- .../xpack/downsample/DownsampleRestIT.java | 10 +++++----- .../xpack/downsample/DownsampleWithBasicRestIT.java | 9 +++++---- .../xpack/downsample/DownsampleWithSecurityRestIT.java | 3 ++- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java index 0f4c205a9a4d9..03753e4bd784b 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleRestIT.java @@ -21,11 +21,11 @@ public class DownsampleRestIT extends ESClientYamlSuiteTestCase { .module("x-pack-downsample") .module("x-pack-ilm") .module("lang-painless") - .module("aggregations") // for auto_date_histogram - .module("mapper-extras") // for scaled_float - .module("x-pack-analytics") // for histogram - .module("data-streams") // for time series - .module("exponential-histogram") + .module("aggregations") // for auto_date_histogram + .module("mapper-extras") // for scaled_float + .module("x-pack-analytics") // for histogram + .module("data-streams") // for time series + .module("exponential-histogram")// for exponential histograms .module("ingest-common") .setting("xpack.license.self_generated.type", "trial") .setting("xpack.security.enabled", "false") diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleWithBasicRestIT.java b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleWithBasicRestIT.java index 5b8379b78a877..c471233ef5e55 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleWithBasicRestIT.java +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleWithBasicRestIT.java @@ -21,11 +21,12 @@ public class DownsampleWithBasicRestIT extends ESClientYamlSuiteTestCase { .module("x-pack-downsample") .module("x-pack-ilm") .module("lang-painless") - .module("aggregations") // for auto_date_histogram - .module("mapper-extras") // for scaled_float - .module("x-pack-analytics") // for histogram - .module("data-streams") // for time series + .module("aggregations") // for auto_date_histogram + .module("mapper-extras") // for scaled_float + .module("x-pack-analytics") // for histogram + .module("data-streams") // for time series .module("ingest-common") + .module("exponential-histogram")// for exponential histograms .setting("xpack.security.enabled", "false") .build(); diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleWithSecurityRestIT.java b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleWithSecurityRestIT.java index 0e8726d880739..0c8f62a8d64f4 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleWithSecurityRestIT.java +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/downsample/DownsampleWithSecurityRestIT.java @@ -25,7 +25,8 @@ public class DownsampleWithSecurityRestIT extends ESClientYamlSuiteTestCase { @ClassRule public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .module("x-pack-downsample") - .module("x-pack-analytics") // for histogram + .module("x-pack-analytics") // for histogram + .module("exponential-histogram")// for exponential histograms .setting("xpack.license.self_generated.type", "trial") .setting("xpack.security.enabled", "true") .user(USERNAME, PASSWORD) From 2c4f4133bda1d140823196149d83b3514865648b Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Wed, 26 Nov 2025 13:42:53 +0200 Subject: [PATCH 7/8] Update x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramMetricFieldProducer.java Co-authored-by: Jonas Kunz --- .../downsample/ExponentialHistogramMetricFieldProducer.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramMetricFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramMetricFieldProducer.java index 0dd53b621f4ce..9d6cda02c4db9 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramMetricFieldProducer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/ExponentialHistogramMetricFieldProducer.java @@ -21,8 +21,7 @@ /** * Class that collects all raw values for an exponential histogram metric field and computes its aggregate (downsampled) - * values. Based on the supported metric types, the subclasses of this class compute values for - * gauge and metric types. + * values. */ final class ExponentialHistogramMetricFieldProducer extends AbstractDownsampleFieldProducer { From 8584a1a65743b5b2afc211d302d7bbb974bf9b7c Mon Sep 17 00:00:00 2001 From: gmarouli Date: Thu, 27 Nov 2025 12:44:55 +0200 Subject: [PATCH 8/8] Add aggregate test --- .../java/org/elasticsearch/xpack/downsample/DownsampleIT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleIT.java b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleIT.java index 3d429bbacbfca..032abdb172140 100644 --- a/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleIT.java +++ b/x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleIT.java @@ -132,6 +132,10 @@ public void testLastValueMethod() throws Exception { downsampleWithSamplingMethod(DownsampleConfig.SamplingMethod.LAST_VALUE); } + public void testAggregateMethod() throws Exception { + downsampleWithSamplingMethod(DownsampleConfig.SamplingMethod.AGGREGATE); + } + private void downsampleWithSamplingMethod(DownsampleConfig.SamplingMethod method) throws Exception { String dataStreamName = "metrics-foo"; String mapping = """