ESQL: load exponential_histogram total count as double instead of long

server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java

@@ -693,7 +693,7 @@ interface ExponentialHistogramBuilder extends Builder {
 
         DoubleBuilder sums();
 
-        LongBuilder valueCounts();
+        DoubleBuilder valueCounts();
 
         DoubleBuilder zeroThresholds();
 

test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java

@@ -775,15 +775,15 @@ private static class ExponentialHistogramBlockBuilder implements BlockLoader.Exp
         private final BlockLoader.DoubleBuilder minima;
         private final BlockLoader.DoubleBuilder maxima;
         private final BlockLoader.DoubleBuilder sums;
-        private final BlockLoader.LongBuilder valueCounts;
+        private final BlockLoader.DoubleBuilder valueCounts;
         private final BlockLoader.DoubleBuilder zeroThresholds;
         private final BlockLoader.BytesRefBuilder encodedHistograms;
 
         private ExponentialHistogramBlockBuilder(BlockLoader.BlockFactory testFactory, int expectedSize) {
             minima = testFactory.doubles(expectedSize);
             maxima = testFactory.doubles(expectedSize);
             sums = testFactory.doubles(expectedSize);
-            valueCounts = testFactory.longs(expectedSize);
+            valueCounts = testFactory.doubles(expectedSize);
             zeroThresholds = testFactory.doubles(expectedSize);
             encodedHistograms = testFactory.bytesRefs(expectedSize);
         }
@@ -838,7 +838,7 @@ public static TestBlock parseHistogramsToBlock(
                     Double max = (Double) maxima.get(i);
                     result.reset(
                         (Double) zeroThresholds.get(i),
-                        (Long) valueCounts.get(i),
+                        ((Double) valueCounts.get(i)).longValue(),
                         sum == null ? 0.0 : sum,
                         min == null ? Double.NaN : min,
                         max == null ? Double.NaN : max,
@@ -887,7 +887,7 @@ public BlockLoader.DoubleBuilder sums() {
         }
 
         @Override
-        public BlockLoader.LongBuilder valueCounts() {
+        public BlockLoader.DoubleBuilder valueCounts() {
             return valueCounts;
         }
 

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java

@@ -509,7 +509,7 @@ public BlockLoader.Block newExponentialHistogramBlockFromDocValues(
         DoubleBlock minima,
         DoubleBlock maxima,
         DoubleBlock sums,
-        LongBlock valueCounts,
+        DoubleBlock valueCounts,
         DoubleBlock zeroThresholds,
         BytesRefBlock encodedHistograms
     ) {

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ExponentialHistogramArrayBlock.java

@@ -46,16 +46,19 @@ final class ExponentialHistogramArrayBlock extends AbstractNonThreadSafeRefCount
     private final DoubleBlock sums;
     /**
      Holds the number of values in each histogram. Note that this is a different concept from getValueCount(position)!
+     At the time of writing, the count will always be an integer.
+     However, as we are planning on eventually supporting extrapolation and rate, the counts will then become fractional.
+     To avoid annoyances with breaking changes later, we store counts as doubles right away.
      */
-    private final LongBlock valueCounts;
+    private final DoubleBlock valueCounts;
     private final DoubleBlock zeroThresholds;
     private final BytesRefBlock encodedHistograms;
 
     ExponentialHistogramArrayBlock(
         DoubleBlock minima,
         DoubleBlock maxima,
         DoubleBlock sums,
-        LongBlock valueCounts,
+        DoubleBlock valueCounts,
         DoubleBlock zeroThresholds,
         BytesRefBlock encodedHistograms
     ) {
@@ -87,7 +90,7 @@ private boolean assertInvariants() {
                 } else {
                     if (b == sums || b == minima || b == maxima) {
                         // sums / minima / maxima should be null exactly when value count is 0 or the histogram is null
-                        assert b.isNull(i) == (valueCounts.getLong(valueCounts.getFirstValueIndex(i)) == 0)
+                        assert b.isNull(i) == (valueCounts.getDouble(valueCounts.getFirstValueIndex(i)) == 0)
                             : "ExponentialHistogramArrayBlock sums/minima/maxima sub-block [" + b + "] has wrong nullity at position " + i;
                     } else {
                         assert b.isNull(i) == false
@@ -107,12 +110,14 @@ private List<Block> getSubBlocks() {
     public ExponentialHistogram getExponentialHistogram(int valueIndex, ExponentialHistogramScratch scratch) {
         BytesRef bytes = encodedHistograms.getBytesRef(encodedHistograms.getFirstValueIndex(valueIndex), scratch.bytesRefScratch);
         double zeroThreshold = zeroThresholds.getDouble(zeroThresholds.getFirstValueIndex(valueIndex));
-        long valueCount = valueCounts.getLong(valueCounts.getFirstValueIndex(valueIndex));
+        double valueCount = valueCounts.getDouble(valueCounts.getFirstValueIndex(valueIndex));
         double sum = valueCount == 0 ? 0.0 : sums.getDouble(sums.getFirstValueIndex(valueIndex));
         double min = valueCount == 0 ? Double.NaN : minima.getDouble(minima.getFirstValueIndex(valueIndex));
         double max = valueCount == 0 ? Double.NaN : maxima.getDouble(maxima.getFirstValueIndex(valueIndex));
         try {
-            scratch.reusedHistogram.reset(zeroThreshold, valueCount, sum, min, max, bytes);
+            // Compressed histograms always have an integral value count, so we can safely round here
+            long roundedValueCount = Math.round(valueCount);
+            scratch.reusedHistogram.reset(zeroThreshold, roundedValueCount, sum, min, max, bytes);
             return scratch.reusedHistogram;
         } catch (IOException e) {
             throw new IllegalStateException("error loading histogram", e);
@@ -138,8 +143,8 @@ public Block buildExponentialHistogramComponentBlock(Component component) {
     public void serializeExponentialHistogram(int valueIndex, SerializedOutput out, BytesRef scratch) {
         // not that this value count is different from getValueCount(position)!
         // this value count represents the number of individual samples the histogram was computed for
-        long valueCount = valueCounts.getLong(valueCounts.getFirstValueIndex(valueIndex));
-        out.appendLong(valueCounts.getLong(valueCounts.getFirstValueIndex(valueIndex)));
+        double valueCount = valueCounts.getDouble(valueCounts.getFirstValueIndex(valueIndex));
+        out.appendDouble(valueCount);
         out.appendDouble(zeroThresholds.getDouble(zeroThresholds.getFirstValueIndex(valueIndex)));
         if (valueCount > 0) {
             // sum / min / max are only non-null for non-empty histograms
@@ -225,7 +230,7 @@ public Block filter(int... positions) {
         DoubleBlock filteredMinima = null;
         DoubleBlock filteredMaxima = null;
         DoubleBlock filteredSums = null;
-        LongBlock filteredValueCounts = null;
+        DoubleBlock filteredValueCounts = null;
         DoubleBlock filteredZeroThresholds = null;
         BytesRefBlock filteredEncodedHistograms = null;
         boolean success = false;
@@ -264,7 +269,7 @@ public Block keepMask(BooleanVector mask) {
         DoubleBlock filteredMinima = null;
         DoubleBlock filteredMaxima = null;
         DoubleBlock filteredSums = null;
-        LongBlock filteredValueCounts = null;
+        DoubleBlock filteredValueCounts = null;
         DoubleBlock filteredZeroThresholds = null;
         BytesRefBlock filteredEncodedHistograms = null;
         boolean success = false;
@@ -320,7 +325,7 @@ public ExponentialHistogramArrayBlock deepCopy(BlockFactory blockFactory) {
         DoubleBlock copiedMinima = null;
         DoubleBlock copiedMaxima = null;
         DoubleBlock copiedSums = null;
-        LongBlock copiedValueCounts = null;
+        DoubleBlock copiedValueCounts = null;
         DoubleBlock copiedZeroThresholds = null;
         BytesRefBlock copiedEncodedHistograms = null;
         boolean success = false;
@@ -361,7 +366,7 @@ public static ExponentialHistogramArrayBlock readFrom(BlockStreamInput in) throw
         DoubleBlock minima = null;
         DoubleBlock maxima = null;
         DoubleBlock sums = null;
-        LongBlock valueCounts = null;
+        DoubleBlock valueCounts = null;
         DoubleBlock zeroThresholds = null;
         BytesRefBlock encodedHistograms = null;
 
@@ -370,7 +375,7 @@ public static ExponentialHistogramArrayBlock readFrom(BlockStreamInput in) throw
             minima = (DoubleBlock) Block.readTypedBlock(in);
             maxima = (DoubleBlock) Block.readTypedBlock(in);
             sums = (DoubleBlock) Block.readTypedBlock(in);
-            valueCounts = (LongBlock) Block.readTypedBlock(in);
+            valueCounts = (DoubleBlock) Block.readTypedBlock(in);
             zeroThresholds = (DoubleBlock) Block.readTypedBlock(in);
             encodedHistograms = (BytesRefBlock) Block.readTypedBlock(in);
             success = true;
@@ -395,7 +400,7 @@ void copyInto(
         DoubleBlock.Builder minimaBuilder,
         DoubleBlock.Builder maximaBuilder,
         DoubleBlock.Builder sumsBuilder,
-        LongBlock.Builder valueCountsBuilder,
+        DoubleBlock.Builder valueCountsBuilder,
         DoubleBlock.Builder zeroThresholdsBuilder,
         BytesRefBlock.Builder encodedHistogramsBuilder,
         int beginInclusive,

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ExponentialHistogramBlockBuilder.java

@@ -22,7 +22,7 @@ public final class ExponentialHistogramBlockBuilder implements ExponentialHistog
     private final DoubleBlock.Builder minimaBuilder;
     private final DoubleBlock.Builder maximaBuilder;
     private final DoubleBlock.Builder sumsBuilder;
-    private final LongBlock.Builder valueCountsBuilder;
+    private final DoubleBlock.Builder valueCountsBuilder;
     private final DoubleBlock.Builder zeroThresholdsBuilder;
     private final BytesRefBlock.Builder encodedHistogramsBuilder;
 
@@ -32,15 +32,15 @@ public final class ExponentialHistogramBlockBuilder implements ExponentialHistog
         DoubleBlock.Builder minimaBuilder = null;
         DoubleBlock.Builder maximaBuilder = null;
         DoubleBlock.Builder sumsBuilder = null;
-        LongBlock.Builder valueCountsBuilder = null;
+        DoubleBlock.Builder valueCountsBuilder = null;
         DoubleBlock.Builder zeroThresholdsBuilder = null;
         BytesRefBlock.Builder encodedHistogramsBuilder = null;
         boolean success = false;
         try {
             minimaBuilder = blockFactory.newDoubleBlockBuilder(estimatedSize);
             maximaBuilder = blockFactory.newDoubleBlockBuilder(estimatedSize);
             sumsBuilder = blockFactory.newDoubleBlockBuilder(estimatedSize);
-            valueCountsBuilder = blockFactory.newLongBlockBuilder(estimatedSize);
+            valueCountsBuilder = blockFactory.newDoubleBlockBuilder(estimatedSize);
             zeroThresholdsBuilder = blockFactory.newDoubleBlockBuilder(estimatedSize);
             encodedHistogramsBuilder = blockFactory.newBytesRefBlockBuilder(estimatedSize);
             this.minimaBuilder = minimaBuilder;
@@ -80,7 +80,7 @@ public BlockLoader.DoubleBuilder sums() {
     }
 
     @Override
-    public BlockLoader.LongBuilder valueCounts() {
+    public BlockLoader.DoubleBuilder valueCounts() {
         return valueCountsBuilder;
     }
 
@@ -131,7 +131,7 @@ public ExponentialHistogramBlockBuilder append(ExponentialHistogram histogram) {
         } else {
             sumsBuilder.appendDouble(histogram.sum());
         }
-        valueCountsBuilder.appendLong(histogram.valueCount());
+        valueCountsBuilder.appendDouble(histogram.valueCount());
         zeroThresholdsBuilder.appendDouble(zeroBucket.zeroThreshold());
         encodedHistogramsBuilder.appendBytesRef(encodedBytes.bytes().toBytesRef());
         return this;
@@ -144,8 +144,8 @@ public ExponentialHistogramBlockBuilder append(ExponentialHistogram histogram) {
      * @param input the input to deserialize from
      */
     public void deserializeAndAppend(ExponentialHistogramBlock.SerializedInput input) {
-        long valueCount = input.readLong();
-        valueCountsBuilder.appendLong(valueCount);
+        double valueCount = input.readDouble();
+        valueCountsBuilder.appendDouble(valueCount);
         zeroThresholdsBuilder.appendDouble(input.readDouble());
         if (valueCount > 0) {
             sumsBuilder.appendDouble(input.readDouble());
@@ -164,7 +164,7 @@ public ExponentialHistogramBlock build() {
         DoubleBlock minima = null;
         DoubleBlock maxima = null;
         DoubleBlock sums = null;
-        LongBlock valueCounts = null;
+        DoubleBlock valueCounts = null;
         DoubleBlock zeroThresholds = null;
         BytesRefBlock encodedHistograms = null;
         boolean success = false;

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/DelegatingBlockLoaderFactory.java

@@ -190,7 +190,7 @@ public BlockLoader.Block buildExponentialHistogramBlockDirect(
             (DoubleBlock) minima,
             (DoubleBlock) maxima,
             (DoubleBlock) sums,
-            (LongBlock) valueCounts,
+            (DoubleBlock) valueCounts,
             (DoubleBlock) zeroThresholds,
             (BytesRefBlock) encodedHistograms
         );

x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/ExponentialHistogramBlockTests.java

@@ -39,7 +39,7 @@ public void testNullSerialization() throws IOException {
             (DoubleBlock) blockFactory().newConstantNullBlock(elementCount),
             (DoubleBlock) blockFactory().newConstantNullBlock(elementCount),
             (DoubleBlock) blockFactory().newConstantNullBlock(elementCount),
-            (LongBlock) blockFactory().newConstantNullBlock(elementCount),
+            (DoubleBlock) blockFactory().newConstantNullBlock(elementCount),
             (DoubleBlock) blockFactory().newConstantNullBlock(elementCount),
             (BytesRefBlock) blockFactory().newConstantNullBlock(elementCount)
         );
@@ -106,7 +106,7 @@ public void testComponentAccess() {
                         case COUNT -> {
                             assertThat(componentBlock.getValueCount(i), equalTo(1));
                             int valueIndex = componentBlock.getFirstValueIndex(i);
-                            assertThat(((LongBlock) componentBlock).getLong(valueIndex), equalTo(histo.valueCount()));
+                            assertThat(((DoubleBlock) componentBlock).getDouble(valueIndex), equalTo((double) histo.valueCount()));
                         }
                     }
                 }

x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java

@@ -430,11 +430,11 @@ protected boolean supportsExponentialHistograms() {
         try {
             return RestEsqlTestCase.hasCapabilities(
                 client(),
-                List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V2.capabilityName())
+                List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V3.capabilityName())
             )
                 && RestEsqlTestCase.hasCapabilities(
                     remoteClusterClient(),
-                    List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V2.capabilityName())
+                    List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V3.capabilityName())
                 );
         } catch (IOException e) {
             throw new RuntimeException(e);

x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java

@@ -58,7 +58,7 @@ protected boolean supportsSourceFieldMapping() {
     protected boolean supportsExponentialHistograms() {
         return RestEsqlTestCase.hasCapabilities(
             client(),
-            List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V2.capabilityName())
+            List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V3.capabilityName())
         );
     }
 

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java

@@ -289,7 +289,7 @@ protected boolean supportsSourceFieldMapping() throws IOException {
     protected boolean supportsExponentialHistograms() {
         return RestEsqlTestCase.hasCapabilities(
             client(),
-            List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V2.capabilityName())
+            List.of(EsqlCapabilities.Cap.EXPONENTIAL_HISTOGRAM_PRE_TECH_PREVIEW_V3.capabilityName())
         );
     }