diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java index 61e17c8ebf9b5..8e5d247aaccb0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java @@ -693,7 +693,7 @@ interface ExponentialHistogramBuilder extends Builder { DoubleBuilder sums(); - LongBuilder valueCounts(); + DoubleBuilder valueCounts(); DoubleBuilder zeroThresholds(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java index 02abb08480865..66fd8619a87d1 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java @@ -775,7 +775,7 @@ private static class ExponentialHistogramBlockBuilder implements BlockLoader.Exp private final BlockLoader.DoubleBuilder minima; private final BlockLoader.DoubleBuilder maxima; private final BlockLoader.DoubleBuilder sums; - private final BlockLoader.LongBuilder valueCounts; + private final BlockLoader.DoubleBuilder valueCounts; private final BlockLoader.DoubleBuilder zeroThresholds; private final BlockLoader.BytesRefBuilder encodedHistograms; @@ -783,7 +783,7 @@ private ExponentialHistogramBlockBuilder(BlockLoader.BlockFactory testFactory, i minima = testFactory.doubles(expectedSize); maxima = testFactory.doubles(expectedSize); sums = testFactory.doubles(expectedSize); - valueCounts = testFactory.longs(expectedSize); + valueCounts = testFactory.doubles(expectedSize); zeroThresholds = testFactory.doubles(expectedSize); encodedHistograms = testFactory.bytesRefs(expectedSize); } @@ -838,7 +838,7 @@ public static TestBlock parseHistogramsToBlock( Double max = (Double) maxima.get(i); result.reset( (Double) zeroThresholds.get(i), - (Long) valueCounts.get(i), + ((Double) valueCounts.get(i)).longValue(), sum == null ? 0.0 : sum, min == null ? Double.NaN : min, max == null ? Double.NaN : max, @@ -887,7 +887,7 @@ public BlockLoader.DoubleBuilder sums() { } @Override - public BlockLoader.LongBuilder valueCounts() { + public BlockLoader.DoubleBuilder valueCounts() { return valueCounts; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java index 39b06dcb684cf..eeab17568e144 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java @@ -509,7 +509,7 @@ public BlockLoader.Block newExponentialHistogramBlockFromDocValues( DoubleBlock minima, DoubleBlock maxima, DoubleBlock sums, - LongBlock valueCounts, + DoubleBlock valueCounts, DoubleBlock zeroThresholds, BytesRefBlock encodedHistograms ) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ExponentialHistogramArrayBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ExponentialHistogramArrayBlock.java index 45b5a922b7022..86170fb6460a9 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ExponentialHistogramArrayBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ExponentialHistogramArrayBlock.java @@ -46,8 +46,11 @@ final class ExponentialHistogramArrayBlock extends AbstractNonThreadSafeRefCount private final DoubleBlock sums; /** Holds the number of values in each histogram. Note that this is a different concept from getValueCount(position)! + At the time of writing, the count will always be an integer. + However, as we are planning on eventually supporting extrapolation and rate, the counts will then become fractional. + To avoid annoyances with breaking changes later, we store counts as doubles right away. */ - private final LongBlock valueCounts; + private final DoubleBlock valueCounts; private final DoubleBlock zeroThresholds; private final BytesRefBlock encodedHistograms; @@ -55,7 +58,7 @@ Holds the number of values in each histogram. Note that this is a different conc DoubleBlock minima, DoubleBlock maxima, DoubleBlock sums, - LongBlock valueCounts, + DoubleBlock valueCounts, DoubleBlock zeroThresholds, BytesRefBlock encodedHistograms ) { @@ -87,7 +90,7 @@ private boolean assertInvariants() { } else { if (b == sums || b == minima || b == maxima) { // sums / minima / maxima should be null exactly when value count is 0 or the histogram is null - assert b.isNull(i) == (valueCounts.getLong(valueCounts.getFirstValueIndex(i)) == 0) + assert b.isNull(i) == (valueCounts.getDouble(valueCounts.getFirstValueIndex(i)) == 0) : "ExponentialHistogramArrayBlock sums/minima/maxima sub-block [" + b + "] has wrong nullity at position " + i; } else { assert b.isNull(i) == false @@ -107,12 +110,14 @@ private List getSubBlocks() { public ExponentialHistogram getExponentialHistogram(int valueIndex, ExponentialHistogramScratch scratch) { BytesRef bytes = encodedHistograms.getBytesRef(encodedHistograms.getFirstValueIndex(valueIndex), scratch.bytesRefScratch); double zeroThreshold = zeroThresholds.getDouble(zeroThresholds.getFirstValueIndex(valueIndex)); - long valueCount = valueCounts.getLong(valueCounts.getFirstValueIndex(valueIndex)); + double valueCount = valueCounts.getDouble(valueCounts.getFirstValueIndex(valueIndex)); double sum = valueCount == 0 ? 0.0 : sums.getDouble(sums.getFirstValueIndex(valueIndex)); double min = valueCount == 0 ? Double.NaN : minima.getDouble(minima.getFirstValueIndex(valueIndex)); double max = valueCount == 0 ? Double.NaN : maxima.getDouble(maxima.getFirstValueIndex(valueIndex)); try { - scratch.reusedHistogram.reset(zeroThreshold, valueCount, sum, min, max, bytes); + // Compressed histograms always have an integral value count, so we can safely round here + long roundedValueCount = Math.round(valueCount); + scratch.reusedHistogram.reset(zeroThreshold, roundedValueCount, sum, min, max, bytes); return scratch.reusedHistogram; } catch (IOException e) { throw new IllegalStateException("error loading histogram", e); @@ -138,8 +143,8 @@ public Block buildExponentialHistogramComponentBlock(Component component) { public void serializeExponentialHistogram(int valueIndex, SerializedOutput out, BytesRef scratch) { // not that this value count is different from getValueCount(position)! // this value count represents the number of individual samples the histogram was computed for - long valueCount = valueCounts.getLong(valueCounts.getFirstValueIndex(valueIndex)); - out.appendLong(valueCounts.getLong(valueCounts.getFirstValueIndex(valueIndex))); + double valueCount = valueCounts.getDouble(valueCounts.getFirstValueIndex(valueIndex)); + out.appendDouble(valueCount); out.appendDouble(zeroThresholds.getDouble(zeroThresholds.getFirstValueIndex(valueIndex))); if (valueCount > 0) { // sum / min / max are only non-null for non-empty histograms @@ -225,7 +230,7 @@ public Block filter(int... positions) { DoubleBlock filteredMinima = null; DoubleBlock filteredMaxima = null; DoubleBlock filteredSums = null; - LongBlock filteredValueCounts = null; + DoubleBlock filteredValueCounts = null; DoubleBlock filteredZeroThresholds = null; BytesRefBlock filteredEncodedHistograms = null; boolean success = false; @@ -264,7 +269,7 @@ public Block keepMask(BooleanVector mask) { DoubleBlock filteredMinima = null; DoubleBlock filteredMaxima = null; DoubleBlock filteredSums = null; - LongBlock filteredValueCounts = null; + DoubleBlock filteredValueCounts = null; DoubleBlock filteredZeroThresholds = null; BytesRefBlock filteredEncodedHistograms = null; boolean success = false; @@ -320,7 +325,7 @@ public ExponentialHistogramArrayBlock deepCopy(BlockFactory blockFactory) { DoubleBlock copiedMinima = null; DoubleBlock copiedMaxima = null; DoubleBlock copiedSums = null; - LongBlock copiedValueCounts = null; + DoubleBlock copiedValueCounts = null; DoubleBlock copiedZeroThresholds = null; BytesRefBlock copiedEncodedHistograms = null; boolean success = false; @@ -361,7 +366,7 @@ public static ExponentialHistogramArrayBlock readFrom(BlockStreamInput in) throw DoubleBlock minima = null; DoubleBlock maxima = null; DoubleBlock sums = null; - LongBlock valueCounts = null; + DoubleBlock valueCounts = null; DoubleBlock zeroThresholds = null; BytesRefBlock encodedHistograms = null; @@ -370,7 +375,7 @@ public static ExponentialHistogramArrayBlock readFrom(BlockStreamInput in) throw minima = (DoubleBlock) Block.readTypedBlock(in); maxima = (DoubleBlock) Block.readTypedBlock(in); sums = (DoubleBlock) Block.readTypedBlock(in); - valueCounts = (LongBlock) Block.readTypedBlock(in); + valueCounts = (DoubleBlock) Block.readTypedBlock(in); zeroThresholds = (DoubleBlock) Block.readTypedBlock(in); encodedHistograms = (BytesRefBlock) Block.readTypedBlock(in); success = true; @@ -395,7 +400,7 @@ void copyInto( DoubleBlock.Builder minimaBuilder, DoubleBlock.Builder maximaBuilder, DoubleBlock.Builder sumsBuilder, - LongBlock.Builder valueCountsBuilder, + DoubleBlock.Builder valueCountsBuilder, DoubleBlock.Builder zeroThresholdsBuilder, BytesRefBlock.Builder encodedHistogramsBuilder, int beginInclusive, diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ExponentialHistogramBlockBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ExponentialHistogramBlockBuilder.java index 67d0c97307acb..9fa64a40e54b1 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ExponentialHistogramBlockBuilder.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ExponentialHistogramBlockBuilder.java @@ -22,7 +22,7 @@ public final class ExponentialHistogramBlockBuilder implements ExponentialHistog private final DoubleBlock.Builder minimaBuilder; private final DoubleBlock.Builder maximaBuilder; private final DoubleBlock.Builder sumsBuilder; - private final LongBlock.Builder valueCountsBuilder; + private final DoubleBlock.Builder valueCountsBuilder; private final DoubleBlock.Builder zeroThresholdsBuilder; private final BytesRefBlock.Builder encodedHistogramsBuilder; @@ -32,7 +32,7 @@ public final class ExponentialHistogramBlockBuilder implements ExponentialHistog DoubleBlock.Builder minimaBuilder = null; DoubleBlock.Builder maximaBuilder = null; DoubleBlock.Builder sumsBuilder = null; - LongBlock.Builder valueCountsBuilder = null; + DoubleBlock.Builder valueCountsBuilder = null; DoubleBlock.Builder zeroThresholdsBuilder = null; BytesRefBlock.Builder encodedHistogramsBuilder = null; boolean success = false; @@ -40,7 +40,7 @@ public final class ExponentialHistogramBlockBuilder implements ExponentialHistog minimaBuilder = blockFactory.newDoubleBlockBuilder(estimatedSize); maximaBuilder = blockFactory.newDoubleBlockBuilder(estimatedSize); sumsBuilder = blockFactory.newDoubleBlockBuilder(estimatedSize); - valueCountsBuilder = blockFactory.newLongBlockBuilder(estimatedSize); + valueCountsBuilder = blockFactory.newDoubleBlockBuilder(estimatedSize); zeroThresholdsBuilder = blockFactory.newDoubleBlockBuilder(estimatedSize); encodedHistogramsBuilder = blockFactory.newBytesRefBlockBuilder(estimatedSize); this.minimaBuilder = minimaBuilder; @@ -80,7 +80,7 @@ public BlockLoader.DoubleBuilder sums() { } @Override - public BlockLoader.LongBuilder valueCounts() { + public BlockLoader.DoubleBuilder valueCounts() { return valueCountsBuilder; } @@ -131,7 +131,7 @@ public ExponentialHistogramBlockBuilder append(ExponentialHistogram histogram) { } else { sumsBuilder.appendDouble(histogram.sum()); } - valueCountsBuilder.appendLong(histogram.valueCount()); + valueCountsBuilder.appendDouble(histogram.valueCount()); zeroThresholdsBuilder.appendDouble(zeroBucket.zeroThreshold()); encodedHistogramsBuilder.appendBytesRef(encodedBytes.bytes().toBytesRef()); return this; @@ -144,8 +144,8 @@ public ExponentialHistogramBlockBuilder append(ExponentialHistogram histogram) { * @param input the input to deserialize from */ public void deserializeAndAppend(ExponentialHistogramBlock.SerializedInput input) { - long valueCount = input.readLong(); - valueCountsBuilder.appendLong(valueCount); + double valueCount = input.readDouble(); + valueCountsBuilder.appendDouble(valueCount); zeroThresholdsBuilder.appendDouble(input.readDouble()); if (valueCount > 0) { sumsBuilder.appendDouble(input.readDouble()); @@ -164,7 +164,7 @@ public ExponentialHistogramBlock build() { DoubleBlock minima = null; DoubleBlock maxima = null; DoubleBlock sums = null; - LongBlock valueCounts = null; + DoubleBlock valueCounts = null; DoubleBlock zeroThresholds = null; BytesRefBlock encodedHistograms = null; boolean success = false; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/DelegatingBlockLoaderFactory.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/DelegatingBlockLoaderFactory.java index 10548ebede2ea..0b819b966ad32 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/DelegatingBlockLoaderFactory.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/DelegatingBlockLoaderFactory.java @@ -190,7 +190,7 @@ public BlockLoader.Block buildExponentialHistogramBlockDirect( (DoubleBlock) minima, (DoubleBlock) maxima, (DoubleBlock) sums, - (LongBlock) valueCounts, + (DoubleBlock) valueCounts, (DoubleBlock) zeroThresholds, (BytesRefBlock) encodedHistograms ); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/ExponentialHistogramBlockTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/ExponentialHistogramBlockTests.java index 4136d7a1b7900..1882bef0c72f3 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/ExponentialHistogramBlockTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/ExponentialHistogramBlockTests.java @@ -39,7 +39,7 @@ public void testNullSerialization() throws IOException { (DoubleBlock) blockFactory().newConstantNullBlock(elementCount), (DoubleBlock) blockFactory().newConstantNullBlock(elementCount), (DoubleBlock) blockFactory().newConstantNullBlock(elementCount), - (LongBlock) blockFactory().newConstantNullBlock(elementCount), + (DoubleBlock) blockFactory().newConstantNullBlock(elementCount), (DoubleBlock) blockFactory().newConstantNullBlock(elementCount), (BytesRefBlock) blockFactory().newConstantNullBlock(elementCount) ); @@ -106,7 +106,7 @@ public void testComponentAccess() { case COUNT -> { assertThat(componentBlock.getValueCount(i), equalTo(1)); int valueIndex = componentBlock.getFirstValueIndex(i); - assertThat(((LongBlock) componentBlock).getLong(valueIndex), equalTo(histo.valueCount())); + assertThat(((DoubleBlock) componentBlock).getDouble(valueIndex), equalTo((double) histo.valueCount())); } } } diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java index 43f8af433e9a1..23b27f851b7e3 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java @@ -430,11 +430,11 @@ protected boolean supportsExponentialHistograms() { try { return RestEsqlTestCase.hasCapabilities( client(), - List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V2.capabilityName()) + List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V3.capabilityName()) ) && RestEsqlTestCase.hasCapabilities( remoteClusterClient(), - List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V2.capabilityName()) + List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V3.capabilityName()) ); } catch (IOException e) { throw new RuntimeException(e); diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java index da85b7964ed66..8147e6d3c934f 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java @@ -58,7 +58,7 @@ protected boolean supportsSourceFieldMapping() { protected boolean supportsExponentialHistograms() { return RestEsqlTestCase.hasCapabilities( client(), - List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V2.capabilityName()) + List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V3.capabilityName()) ); } diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java index e8b8e26c74e5a..3e31778fab3f3 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java @@ -289,7 +289,7 @@ protected boolean supportsSourceFieldMapping() throws IOException { protected boolean supportsExponentialHistograms() { return RestEsqlTestCase.hasCapabilities( client(), - List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V2.capabilityName()) + List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V3.capabilityName()) ); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/exponential_histogram.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/exponential_histogram.csv-spec index 2a0a0c73b8021..a6d1e655de8e5 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/exponential_histogram.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/exponential_histogram.csv-spec @@ -1,5 +1,5 @@ loadFiltered -required_capability: exponential_histogram_pre_tech_preview_v2 +required_capability: exponential_histogram_pre_tech_preview_v3 FROM exp_histo_sample | WHERE STARTS_WITH(instance, "dummy") | SORT instance | KEEP instance, responseTime ; @@ -17,7 +17,7 @@ dummy-zero_threshold_only | "{""scale"":0,""zero"":{""threshold"":2.0E-5}}" allAggsGrouped -required_capability: exponential_histogram_pre_tech_preview_v2 +required_capability: exponential_histogram_pre_tech_preview_v3 FROM exp_histo_sample | EVAL instance = CASE(STARTS_WITH(instance, "dummy"), "dummy-grouped", instance) @@ -36,7 +36,7 @@ instance-2 | 2.2E-4 | 2.744054 | 6.469E-4 | 0.0016068 | 27.706 allAggsFiltered -required_capability: exponential_histogram_pre_tech_preview_v2 +required_capability: exponential_histogram_pre_tech_preview_v3 FROM exp_histo_sample | STATS min = MIN(responseTime) WHERE instance == "instance-0", @@ -55,7 +55,7 @@ min:double | max:double | median:double | p75:double | sum:double | avg:double allAggsInlineGrouped -required_capability: exponential_histogram_pre_tech_preview_v2 +required_capability: exponential_histogram_pre_tech_preview_v3 FROM exp_histo_sample | INLINE STATS min = MIN(responseTime), max = MAX(responseTime), median = MEDIAN(responseTime), p75 = PERCENTILE(responseTime,75), sum = SUM(responseTime), avg = AVG(responseTime) BY instance @@ -86,7 +86,7 @@ instance-0 | 2.4E-4 | 6.786232 | 0.0211404 | 0.2608237 allAggsOnEmptyHistogram -required_capability: exponential_histogram_pre_tech_preview_v2 +required_capability: exponential_histogram_pre_tech_preview_v3 FROM exp_histo_sample | WHERE instance == "dummy-empty" | STATS min = MIN(responseTime), max = MAX(responseTime), median = MEDIAN(responseTime), p75 = PERCENTILE(responseTime,75), sum = SUM(responseTime), avg = AVG(responseTime) @@ -99,7 +99,7 @@ NULL | NULL | NULL | NULL | NULL | NULL histoAsCaseValue -required_capability: exponential_histogram_pre_tech_preview_v2 +required_capability: exponential_histogram_pre_tech_preview_v3 FROM exp_histo_sample | INLINE STATS p50 = PERCENTILE(responseTime, 50) BY instance, @timestamp @@ -113,7 +113,7 @@ filteredCount:long ; ungroupedPercentiles -required_capability: exponential_histogram_pre_tech_preview_v2 +required_capability: exponential_histogram_pre_tech_preview_v3 FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy") | STATS p0 = PERCENTILE(responseTime,0), p50 = PERCENTILE(responseTime,50), p99 = PERCENTILE(responseTime, 99), p100 = PERCENTILE(responseTime,100) @@ -128,7 +128,7 @@ p0:double | p50:double | p99:double | p100:double groupedPercentiles -required_capability: exponential_histogram_pre_tech_preview_v2 +required_capability: exponential_histogram_pre_tech_preview_v3 FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy") | STATS p0 = PERCENTILE(responseTime,0), p50 = PERCENTILE(responseTime,50), p99 = PERCENTILE(responseTime, 99), p100 = PERCENTILE(responseTime,100) BY instance @@ -146,7 +146,7 @@ instance-2 | 2.2E-4 | 6.469E-4 | 0.0857672 | 2.7059714542564097 percentileOnEmptyHistogram -required_capability: exponential_histogram_pre_tech_preview_v2 +required_capability: exponential_histogram_pre_tech_preview_v3 FROM exp_histo_sample | WHERE instance == "dummy-empty" | STATS p50 = PERCENTILE(responseTime,50) @@ -160,7 +160,7 @@ NULL ungroupedMinMax -required_capability: exponential_histogram_pre_tech_preview_v2 +required_capability: exponential_histogram_pre_tech_preview_v3 FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy") | STATS min = MIN(responseTime), max = MAX(responseTime) @@ -174,7 +174,7 @@ min:double | max:double groupedMinMax -required_capability: exponential_histogram_pre_tech_preview_v2 +required_capability: exponential_histogram_pre_tech_preview_v3 FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy") | STATS min = MIN(responseTime), max = MAX(responseTime) BY instance @@ -191,7 +191,7 @@ instance-2 | 2.2E-4 | 2.744054 minMaxOnEmptyHistogram -required_capability: exponential_histogram_pre_tech_preview_v2 +required_capability: exponential_histogram_pre_tech_preview_v3 FROM exp_histo_sample | WHERE instance == "dummy-empty" | STATS min = MIN(responseTime), max = MAX(responseTime) @@ -204,7 +204,7 @@ NULL | NULL ungroupedAvg -required_capability: exponential_histogram_pre_tech_preview_v2 +required_capability: exponential_histogram_pre_tech_preview_v3 FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy") | STATS avg = AVG(responseTime) @@ -217,7 +217,7 @@ avg:double groupedAvg -required_capability: exponential_histogram_pre_tech_preview_v2 +required_capability: exponential_histogram_pre_tech_preview_v3 FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy") | STATS avg = AVG(responseTime) BY instance @@ -233,7 +233,7 @@ instance-2 | 0.008197047633136096 avgOnEmptyHistogram -required_capability: exponential_histogram_pre_tech_preview_v2 +required_capability: exponential_histogram_pre_tech_preview_v3 FROM exp_histo_sample | WHERE instance == "dummy-empty" | STATS avg = AVG(responseTime) @@ -246,7 +246,7 @@ NULL ungroupedSum -required_capability: exponential_histogram_pre_tech_preview_v2 +required_capability: exponential_histogram_pre_tech_preview_v3 FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy") | STATS sum = SUM(responseTime) @@ -259,7 +259,7 @@ sum:double groupedSum -required_capability: exponential_histogram_pre_tech_preview_v2 +required_capability: exponential_histogram_pre_tech_preview_v3 FROM exp_histo_sample | WHERE NOT STARTS_WITH(instance, "dummy") | STATS sum = SUM(responseTime) BY instance diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 8d82b143360dd..070330957eff8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1558,7 +1558,7 @@ public enum Cap { * When implementing changes on this type, we'll simply increment the version suffix at the end to prevent bwc tests from running. * As soon as we move into tech preview, we'll replace this capability with a "EXPONENTIAL_HISTOGRAM_TECH_PREVIEW" one. */ - EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V2(EXPONENTIAL_HISTOGRAM_FEATURE_FLAG), + EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V3(EXPONENTIAL_HISTOGRAM_FEATURE_FLAG), /** * Create new block when filtering OrdinalBytesRefBlock diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/ExtractHistogramComponent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/ExtractHistogramComponent.java index 1f1bf6fdc62e7..3181f0ed682bd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/ExtractHistogramComponent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/ExtractHistogramComponent.java @@ -39,7 +39,6 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; -import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; /** * Extracts a {@link org.elasticsearch.compute.data.ExponentialHistogramBlock.Component} from an exponential histogram. @@ -65,7 +64,7 @@ public class ExtractHistogramComponent extends EsqlScalarFunction { * @param componentOrdinal The {@link org.elasticsearch.compute.data.ExponentialHistogramBlock.Component#ordinal()} * as integer-expression, must be foldable */ - @FunctionInfo(returnType = { "long", "double" }) + @FunctionInfo(returnType = { "double" }) public ExtractHistogramComponent( Source source, @Param(name = "histogram", type = { "exponential_histogram" }) Expression field, @@ -119,8 +118,7 @@ public String getWriteableName() { public DataType dataType() { return switch (component()) { case null -> DataType.NULL; - case MIN, MAX, SUM -> DOUBLE; - case COUNT -> LONG; + case MIN, MAX, SUM, COUNT -> DOUBLE; }; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/ExtractHistogramComponentTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/ExtractHistogramComponentTests.java index 4e8b8612538fb..09f56ffd2d50a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/ExtractHistogramComponentTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/histogram/ExtractHistogramComponentTests.java @@ -61,7 +61,7 @@ public static Iterable parameters() { return new TestCaseSupplier.TestCase( List.of(histogram, componentOrdinalSupplier.get()), "ExtractHistogramComponentEvaluator[field=Attribute[channel=0],component=" + component + "]", - getExpectedDataTypeForComponent(component), + DataType.DOUBLE, equalTo(getExpectedValue(histogram, component)) ); } @@ -92,14 +92,7 @@ private static Object getExpectedValue(TestCaseSupplier.TypedData histogram, Exp yield Double.isNaN(max) ? null : max; } case SUM -> value.valueCount() > 0 ? value.sum() : null; - case COUNT -> value.valueCount(); - }; - } - - private static DataType getExpectedDataTypeForComponent(ExponentialHistogramBlock.Component component) { - return switch (component) { - case MIN, MAX, SUM -> DataType.DOUBLE; - case COUNT -> DataType.LONG; + case COUNT -> (double) value.valueCount(); }; } diff --git a/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java b/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java index 5da5283f55dd4..f76cc0033c3ea 100644 --- a/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java +++ b/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java @@ -51,7 +51,6 @@ import org.elasticsearch.index.mapper.blockloader.docvalues.BlockDocValuesReader; import org.elasticsearch.index.mapper.blockloader.docvalues.BytesRefsFromBinaryBlockLoader; import org.elasticsearch.index.mapper.blockloader.docvalues.DoublesBlockLoader; -import org.elasticsearch.index.mapper.blockloader.docvalues.LongsBlockLoader; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; import org.elasticsearch.search.DocValueFormat; @@ -355,7 +354,8 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { DoublesBlockLoader minimaLoader = new DoublesBlockLoader(valuesMinSubFieldName(name()), NumericUtils::sortableLongToDouble); DoublesBlockLoader maximaLoader = new DoublesBlockLoader(valuesMaxSubFieldName(name()), NumericUtils::sortableLongToDouble); DoublesBlockLoader sumsLoader = new DoublesBlockLoader(valuesSumSubFieldName(name()), NumericUtils::sortableLongToDouble); - LongsBlockLoader valueCountsLoader = new LongsBlockLoader(valuesCountSubFieldName(name())); + // we store the counts as integers for better compression, but the block requires doubles. So we simply cast the value + DoublesBlockLoader valueCountsLoader = new DoublesBlockLoader(valuesCountSubFieldName(name()), longVal -> (double) longVal); DoublesBlockLoader zeroThresholdsLoader = new DoublesBlockLoader( zeroThresholdSubFieldName(name()), NumericUtils::sortableLongToDouble