From 47df1d4795fe292324736d106a731de7e081030e Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Wed, 19 Jun 2019 09:30:01 +0200
Subject: [PATCH 01/19] Add new storage strategy when cardinality if points is
 low on a BKD tree leaf

---
 .../org/apache/lucene/util/bkd/BKDReader.java | 82 +++++++++++++++++--
 .../org/apache/lucene/util/bkd/BKDWriter.java | 72 +++++++++++++---
 2 files changed, 136 insertions(+), 18 deletions(-)

diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
index c1f5a63d7de8..4aa6fa03b30e 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
@@ -441,8 +441,15 @@ int readDocIDs(IndexInput in, long blockFP, int[] docIDs) throws IOException {
 
   void visitDocValues(int[] commonPrefixLengths, byte[] scratchDataPackedValue, byte[] scratchMinIndexPackedValue, byte[] scratchMaxIndexPackedValue,
                       IndexInput in, int[] docIDs, int count, IntersectVisitor visitor) throws IOException {
+    if (version >= BKDWriter.VERSION_LOW_CARDINALITY_LEAVES) {
+      visitDocValuesWithCardinality(commonPrefixLengths, scratchDataPackedValue, scratchMinIndexPackedValue, scratchMaxIndexPackedValue, in, docIDs, count, visitor);
+    } else {
+      visitDocValuesNoCardinality(commonPrefixLengths, scratchDataPackedValue, scratchMinIndexPackedValue, scratchMaxIndexPackedValue, in, docIDs, count, visitor);
+    }
+  }
 
-
+  void visitDocValuesNoCardinality(int[] commonPrefixLengths, byte[] scratchDataPackedValue, byte[] scratchMinIndexPackedValue, byte[] scratchMaxIndexPackedValue,
+                      IndexInput in, int[] docIDs, int count, IntersectVisitor visitor) throws IOException {
     readCommonPrefixes(commonPrefixLengths, scratchDataPackedValue, in);
 
     if (numIndexDims != 1 && version >= BKDWriter.VERSION_LEAF_STORES_BOUNDS) {
@@ -480,12 +487,62 @@ void visitDocValues(int[] commonPrefixLengths, byte[] scratchDataPackedValue, by
     int compressedDim = readCompressedDim(in);
 
     if (compressedDim == -1) {
-      visitRawDocValues(commonPrefixLengths, scratchDataPackedValue, in, docIDs, count, visitor);
+      visitUniqueRawDocValues(scratchDataPackedValue, docIDs, count, visitor);
     } else {
       visitCompressedDocValues(commonPrefixLengths, scratchDataPackedValue, in, docIDs, count, visitor, compressedDim);
     }
   }
 
+  void visitDocValuesWithCardinality(int[] commonPrefixLengths, byte[] scratchDataPackedValue, byte[] scratchMinIndexPackedValue, byte[] scratchMaxIndexPackedValue,
+                                     IndexInput in, int[] docIDs, int count, IntersectVisitor visitor) throws IOException {
+
+    readCommonPrefixes(commonPrefixLengths, scratchDataPackedValue, in);
+    int compressedDim = readCompressedDim(in);
+    if (compressedDim == -1) {
+      //all values are the same
+      visitor.grow(count);
+      visitUniqueRawDocValues(scratchDataPackedValue, docIDs, count, visitor);
+    } else {
+      if (numIndexDims != 1 && version >= BKDWriter.VERSION_LEAF_STORES_BOUNDS) {
+        byte[] minPackedValue = scratchMinIndexPackedValue;
+        System.arraycopy(scratchDataPackedValue, 0, minPackedValue, 0, packedIndexBytesLength);
+        byte[] maxPackedValue = scratchMaxIndexPackedValue;
+        //Copy common prefixes before reading adjusted
+        // box
+        System.arraycopy(minPackedValue, 0, maxPackedValue, 0, packedIndexBytesLength);
+        readMinMax(commonPrefixLengths, minPackedValue, maxPackedValue, in);
+
+        // The index gives us range of values for each dimension, but the actual range of values
+        // might be much more narrow than what the index told us, so we double check the relation
+        // here, which is cheap yet might help figure out that the block either entirely matches
+        // or does not match at all. This is especially more likely in the case that there are
+        // multiple dimensions that have correlation, ie. splitting on one dimension also
+        // significantly changes the range of values in another dimension.
+        Relation r = visitor.compare(minPackedValue, maxPackedValue);
+        if (r == Relation.CELL_OUTSIDE_QUERY) {
+          return;
+        }
+        visitor.grow(count);
+
+        if (r == Relation.CELL_INSIDE_QUERY) {
+          for (int i = 0; i < count; ++i) {
+            visitor.visit(docIDs[i]);
+          }
+          return;
+        }
+      } else {
+        visitor.grow(count);
+      }
+      if (compressedDim == -2) {
+        //low cardinality values
+        visitSparseRawDocValues(commonPrefixLengths, scratchDataPackedValue, in, docIDs, count, visitor);
+      } else {
+        //high cardinality
+        visitCompressedDocValues(commonPrefixLengths, scratchDataPackedValue, in, docIDs, count, visitor, compressedDim);
+      }
+    }
+  }
+
   private void readMinMax(int[] commonPrefixLengths, byte[] minPackedValue, byte[] maxPackedValue, IndexInput in) throws IOException {
     for (int dim = 0; dim < numIndexDims; dim++) {
       int prefix = commonPrefixLengths[dim];
@@ -495,12 +552,27 @@ private void readMinMax(int[] commonPrefixLengths, byte[] minPackedValue, byte[]
   }
 
   // Just read suffixes for every dimension
-  private void visitRawDocValues(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in, int[] docIDs, int count, IntersectVisitor visitor) throws IOException {
-    for (int i = 0; i < count; ++i) {
+  private void visitSparseRawDocValues(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in, int[] docIDs, int count, IntersectVisitor visitor) throws IOException {
+    int i;
+    for (i = 0; i < count;) {
+      int length = in.readVInt();
       for(int dim=0;dim<numDataDims;dim++) {
         int prefix = commonPrefixLengths[dim];
         in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix);
       }
+      for (int j = i; j < i + length; j++) {
+        visitor.visit(docIDs[j], scratchPackedValue);
+      }
+      i+= length;
+    }
+    if (i != count) {
+      throw new CorruptIndexException("Sub blocks do not add up to the expected count: " + count + " != " + i, in);
+    }
+  }
+
+  // Just read suffixes for every dimension
+  private void visitUniqueRawDocValues(byte[] scratchPackedValue, int[] docIDs, int count, IntersectVisitor visitor) throws IOException {
+    for (int i = 0; i < count; i++) {
       visitor.visit(docIDs[i], scratchPackedValue);
     }
   }
@@ -530,7 +602,7 @@ private void visitCompressedDocValues(int[] commonPrefixLengths, byte[] scratchP
 
   private int readCompressedDim(IndexInput in) throws IOException {
     int compressedDim = in.readByte();
-    if (compressedDim < -1 || compressedDim >= numDataDims) {
+    if (compressedDim < -2 || compressedDim >= numDataDims) {
       throw new CorruptIndexException("Got compressedDim="+compressedDim, in);
     }
     return compressedDim;
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index 659935cc9292..16a865cb5959 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -80,7 +80,8 @@ public class BKDWriter implements Closeable {
   //public static final int VERSION_CURRENT = VERSION_START;
   public static final int VERSION_LEAF_STORES_BOUNDS = 5;
   public static final int VERSION_SELECTIVE_INDEXING = 6;
-  public static final int VERSION_CURRENT = VERSION_SELECTIVE_INDEXING;
+  public static final int VERSION_LOW_CARDINALITY_LEAVES= 7;
+  public static final int VERSION_CURRENT = VERSION_LOW_CARDINALITY_LEAVES;
 
   /** How many bytes each docs takes in the fixed-width offline format */
   private final int bytesPerDoc;
@@ -516,6 +517,7 @@ private class OneDimensionBKDWriter {
     final int[] leafDocs = new int[maxPointsInLeafNode];
     private long valueCount;
     private int leafCount;
+    private int leafCardinality;
 
     OneDimensionBKDWriter(IndexOutput out) {
       if (numIndexDims != 1) {
@@ -546,6 +548,9 @@ void add(byte[] packedValue, int docID) throws IOException {
       assert valueInOrder(valueCount + leafCount,
           0, lastPackedValue, packedValue, 0, docID, lastDocID);
 
+      if (leafCount == 0 || Arrays.mismatch(leafValues, (leafCount - 1) * bytesPerDim, leafCount * bytesPerDim, packedValue, 0, bytesPerDim) != -1) {
+        leafCardinality++;
+      }
       System.arraycopy(packedValue, 0, leafValues, leafCount * packedBytesLength, packedBytesLength);
       leafDocs[leafCount] = docID;
       docsSeen.set(docID);
@@ -558,7 +563,8 @@ assert valueInOrder(valueCount + leafCount,
       if (leafCount == maxPointsInLeafNode) {
         // We write a block once we hit exactly the max count ... this is different from
         // when we write N > 1 dimensional points where we write between max/2 and max per leaf block
-        writeLeafBlock();
+        writeLeafBlock(leafCardinality);
+        leafCardinality = 0;
         leafCount = 0;
       }
 
@@ -567,7 +573,8 @@ assert valueInOrder(valueCount + leafCount,
 
     public long finish() throws IOException {
       if (leafCount > 0) {
-        writeLeafBlock();
+        writeLeafBlock(leafCardinality);
+        leafCardinality = 0;
         leafCount = 0;
       }
 
@@ -593,7 +600,7 @@ public long finish() throws IOException {
       return indexFP;
     }
 
-    private void writeLeafBlock() throws IOException {
+    private void writeLeafBlock(int leafCardinality) throws IOException {
       assert leafCount != 0;
       if (valueCount == 0) {
         System.arraycopy(leafValues, 0, minPackedValue, 0, packedIndexBytesLength);
@@ -613,7 +620,7 @@ private void writeLeafBlock() throws IOException {
       int offset = (leafCount - 1) * packedBytesLength;
       int prefix = Arrays.mismatch(leafValues, 0, bytesPerDim, leafValues, offset, offset + bytesPerDim);
       if (prefix == -1) {
-          prefix = bytesPerDim;
+        prefix = bytesPerDim;
       }
 
       commonPrefixLengths[0] = prefix;
@@ -635,7 +642,7 @@ public BytesRef apply(int i) {
       assert valuesInOrderAndBounds(leafCount, 0, ArrayUtil.copyOfSubArray(leafValues, 0, packedBytesLength),
           ArrayUtil.copyOfSubArray(leafValues, (leafCount - 1) * packedBytesLength, leafCount * packedBytesLength),
           packedValues, leafDocs, 0);
-      writeLeafBlockPackedValues(scratchOut, commonPrefixLengths, leafCount, 0, packedValues);
+      writeLeafBlockPackedValues(scratchOut, commonPrefixLengths, leafCount, 0, packedValues, leafCardinality);
       scratchOut.copyTo(out);
       scratchOut.reset();
     }
@@ -1028,17 +1035,43 @@ private void writeLeafBlockDocs(DataOutput out, int[] docIDs, int start, int cou
     DocIdsWriter.writeDocIds(docIDs, start, count, out);
   }
 
-  private void writeLeafBlockPackedValues(DataOutput out, int[] commonPrefixLengths, int count, int sortedDim, IntFunction<BytesRef> packedValues) throws IOException {
+  private void writeLeafBlockPackedValues(DataOutput out, int[] commonPrefixLengths, int count, int sortedDim, IntFunction<BytesRef> packedValues, int leafCardinality) throws IOException {
     int prefixLenSum = Arrays.stream(commonPrefixLengths).sum();
     if (prefixLenSum == packedBytesLength) {
       // all values in this block are equal
       out.writeByte((byte) -1);
-    } else {
+    } else if (leafCardinality * (prefixLenSum + 2)  <= count * prefixLenSum) {
+      //estimate if storing the values with cardinality is cheaper than storing all values
+      out.writeByte((byte) -2);
       if (numIndexDims != 1) {
         writeActualBounds(out, commonPrefixLengths, count, packedValues);
       }
+      BytesRef value = packedValues.apply(0);
+      System.arraycopy(value.bytes, value.offset, scratch1, 0, packedBytesLength);
+      int cardinality = 1;
+      for (int i = 1; i < count; i++) {
+        value = packedValues.apply(i);
+        if (Arrays.mismatch(value.bytes, value.offset, value.offset + value.length, scratch1, 0, packedBytesLength) != -1) {
+          out.writeVInt(cardinality);
+          for(int j = 0; j < numDataDims; j++) {
+            out.writeBytes(scratch1, j * bytesPerDim + commonPrefixLengths[j], bytesPerDim - commonPrefixLengths[j]);
+          }
+          System.arraycopy(value.bytes, value.offset, scratch1, 0, packedBytesLength);
+          cardinality = 1;
+        } else {
+          cardinality++;
+        }
+      }
+      out.writeVInt(cardinality);
+      for(int i = 0; i < numDataDims; i++) {
+        out.writeBytes(scratch1, i * bytesPerDim + commonPrefixLengths[i], bytesPerDim - commonPrefixLengths[i]);
+      }
+    } else {
       assert commonPrefixLengths[sortedDim] < bytesPerDim;
       out.writeByte((byte) sortedDim);
+      if (numIndexDims != 1) {
+        writeActualBounds(out, commonPrefixLengths, count, packedValues);
+      }
       int compressedByteOffset = sortedDim * bytesPerDim + commonPrefixLengths[sortedDim];
       commonPrefixLengths[sortedDim]++;
       for (int i = 0; i < count; ) {
@@ -1246,11 +1279,17 @@ private void build(int nodeID, int leafNodeOffset,
       final int count = to - from;
       assert count <= maxPointsInLeafNode;
 
-      // Compute common prefixes
+      // Compute common prefixes and cardinality
       Arrays.fill(commonPrefixLengths, bytesPerDim);
       reader.getValue(from, scratchBytesRef1);
+      int leafCardinality = 1;
+      System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset, scratch1, 0, packedBytesLength);
       for (int i = from + 1; i < to; ++i) {
         reader.getValue(i, scratchBytesRef2);
+        if (Arrays.mismatch(scratch1, 0, packedBytesLength, scratchBytesRef2.bytes, scratchBytesRef2.offset, scratchBytesRef2.offset + packedBytesLength) != -1) {
+          leafCardinality++;
+          System.arraycopy(scratchBytesRef2.bytes, scratchBytesRef2.offset, scratch1, 0, packedBytesLength);
+        }
         for (int dim=0;dim<numDataDims;dim++) {
           final int offset = dim * bytesPerDim;
           int dimensionPrefixLength = commonPrefixLengths[dim];
@@ -1323,7 +1362,7 @@ public BytesRef apply(int i) {
       };
       assert valuesInOrderAndBounds(count, sortedDim, minPackedValue, maxPackedValue, packedValues,
           docIDs, 0);
-      writeLeafBlockPackedValues(scratchOut, commonPrefixLengths, count, sortedDim, packedValues);
+      writeLeafBlockPackedValues(scratchOut, commonPrefixLengths, count, sortedDim, packedValues, leafCardinality);
       scratchOut.copyTo(out);
       scratchOut.reset();
     } else {
@@ -1395,7 +1434,7 @@ private void build(int nodeID, int leafNodeOffset,
       int from = Math.toIntExact(points.start);
       int to = Math.toIntExact(points.start + points.count);
       //we store common prefix on scratch1
-      computeCommonPrefixLength(heapSource, scratch1, from, to);
+      int leafCardinality = computeCommonPrefixLength(heapSource, scratch1, from, to);
 
       int sortedDim = 0;
       int sortedDimCardinality = Integer.MAX_VALUE;
@@ -1459,7 +1498,7 @@ public BytesRef apply(int i) {
       };
       assert valuesInOrderAndBounds(count, sortedDim, minPackedValue, maxPackedValue, packedValues,
           heapSource.docIDs, from);
-      writeLeafBlockPackedValues(out, commonPrefixLengths, count, sortedDim, packedValues);
+      writeLeafBlockPackedValues(out, commonPrefixLengths, count, sortedDim, packedValues, leafCardinality);
 
     } else {
       // Inner node: partition/recurse
@@ -1516,16 +1555,22 @@ assert valuesInOrderAndBounds(count, sortedDim, minPackedValue, maxPackedValue,
     }
   }
 
-  private void computeCommonPrefixLength(HeapPointWriter heapPointWriter, byte[] commonPrefix, int from, int to) {
+  private int computeCommonPrefixLength(HeapPointWriter heapPointWriter, byte[] commonPrefix, int from, int to) {
     Arrays.fill(commonPrefixLengths, bytesPerDim);
     PointValue value = heapPointWriter.getPackedValueSlice(from);
     BytesRef packedValue = value.packedValue();
     for (int dim = 0; dim < numDataDims; dim++) {
       System.arraycopy(packedValue.bytes, packedValue.offset + dim * bytesPerDim, commonPrefix, dim * bytesPerDim, bytesPerDim);
     }
+    System.arraycopy(packedValue.bytes, packedValue.offset, scratch2, 0, packedBytesLength);
+    int leafCardinality = 1;
     for (int i = from + 1; i < to; i++) {
       value =  heapPointWriter.getPackedValueSlice(i);
       packedValue = value.packedValue();
+      if (Arrays.mismatch(scratch2, 0, packedBytesLength, packedValue.bytes, packedValue.offset, packedValue.offset + packedBytesLength) != -1) {
+        leafCardinality++;
+        System.arraycopy(packedValue.bytes, packedValue.offset, scratch2, 0, packedBytesLength);
+      }
       for (int dim = 0; dim < numDataDims; dim++) {
         if (commonPrefixLengths[dim] != 0) {
           int j = Arrays.mismatch(commonPrefix, dim * bytesPerDim, dim * bytesPerDim + commonPrefixLengths[dim], packedValue.bytes, packedValue.offset + dim * bytesPerDim, packedValue.offset + dim * bytesPerDim + commonPrefixLengths[dim]);
@@ -1535,6 +1580,7 @@ private void computeCommonPrefixLength(HeapPointWriter heapPointWriter, byte[] c
         }
       }
     }
+    return leafCardinality;
   }
 
   // only called from assert

From 1f8738e65d559d44c3032061286123766f75d5ac Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Wed, 19 Jun 2019 09:46:14 +0200
Subject: [PATCH 02/19] Compute correctly when to perform the optimisation

---
 lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index 16a865cb5959..8641d620237e 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -1040,7 +1040,7 @@ private void writeLeafBlockPackedValues(DataOutput out, int[] commonPrefixLength
     if (prefixLenSum == packedBytesLength) {
       // all values in this block are equal
       out.writeByte((byte) -1);
-    } else if (leafCardinality * (prefixLenSum + 2)  <= count * prefixLenSum) {
+    } else if (leafCardinality * (packedBytesLength - prefixLenSum + 2)  <= count * (packedBytesLength - prefixLenSum)) {
       //estimate if storing the values with cardinality is cheaper than storing all values
       out.writeByte((byte) -2);
       if (numIndexDims != 1) {

From 69f4d4e0010e1e8b5b8f9b25feeca06c85f4ac33 Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Thu, 20 Jun 2019 10:27:24 +0200
Subject: [PATCH 03/19] Change computation of low cardinality to consider the
 size of cardinality as 1 byte

---
 lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index 8641d620237e..95ecfb47fd5c 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -1040,7 +1040,7 @@ private void writeLeafBlockPackedValues(DataOutput out, int[] commonPrefixLength
     if (prefixLenSum == packedBytesLength) {
       // all values in this block are equal
       out.writeByte((byte) -1);
-    } else if (leafCardinality * (packedBytesLength - prefixLenSum + 2)  <= count * (packedBytesLength - prefixLenSum)) {
+    } else if (leafCardinality * (packedBytesLength - prefixLenSum + 1)  <= count * (packedBytesLength - prefixLenSum)) {
       //estimate if storing the values with cardinality is cheaper than storing all values
       out.writeByte((byte) -2);
       if (numIndexDims != 1) {

From 033564b12a92611d6a5077cbec2b65f23867443c Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Thu, 20 Jun 2019 10:30:25 +0200
Subject: [PATCH 04/19] Fail if commpressed dim is -2 and we are on an earlier
 version

---
 lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
index 4aa6fa03b30e..c8230c6a56b7 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
@@ -602,7 +602,7 @@ private void visitCompressedDocValues(int[] commonPrefixLengths, byte[] scratchP
 
   private int readCompressedDim(IndexInput in) throws IOException {
     int compressedDim = in.readByte();
-    if (compressedDim < -2 || compressedDim >= numDataDims) {
+    if (compressedDim < -2 || compressedDim >= numDataDims || (version < BKDWriter.VERSION_LOW_CARDINALITY_LEAVES && compressedDim == -2)) {
       throw new CorruptIndexException("Got compressedDim="+compressedDim, in);
     }
     return compressedDim;

From 15e8958233faeb2f6dc29d496765bc16d3646db8 Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Fri, 21 Jun 2019 14:23:04 +0200
Subject: [PATCH 05/19] fix the formula to decide to use the new storage
 strategy. It takes into account the runLen compression.

---
 .../core/src/java/org/apache/lucene/util/bkd/BKDWriter.java  | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index 95ecfb47fd5c..7c67ab0614f7 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -1040,8 +1040,9 @@ private void writeLeafBlockPackedValues(DataOutput out, int[] commonPrefixLength
     if (prefixLenSum == packedBytesLength) {
       // all values in this block are equal
       out.writeByte((byte) -1);
-    } else if (leafCardinality * (packedBytesLength - prefixLenSum + 1)  <= count * (packedBytesLength - prefixLenSum)) {
-      //estimate if storing the values with cardinality is cheaper than storing all values
+    } else if (leafCardinality * (packedBytesLength - prefixLenSum + 1)  <= count * (packedBytesLength - prefixLenSum - 1)) {
+      //estimate if storing the values with cardinality is cheaper than storing all values.
+      //+1 is the byte needed for storing the cardinality and -1 is the runLen compression.
       out.writeByte((byte) -2);
       if (numIndexDims != 1) {
         writeActualBounds(out, commonPrefixLengths, count, packedValues);

From f39370c0b7ee4d5a055f87953fe661036c88399a Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Sun, 23 Jun 2019 07:39:06 +0200
Subject: [PATCH 06/19] compute exactlay the cost of runLen compression

---
 .../org/apache/lucene/util/bkd/BKDWriter.java | 112 +++++++++++-------
 1 file changed, 70 insertions(+), 42 deletions(-)

diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index 7c67ab0614f7..a925e852d926 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -1040,54 +1040,82 @@ private void writeLeafBlockPackedValues(DataOutput out, int[] commonPrefixLength
     if (prefixLenSum == packedBytesLength) {
       // all values in this block are equal
       out.writeByte((byte) -1);
-    } else if (leafCardinality * (packedBytesLength - prefixLenSum + 1)  <= count * (packedBytesLength - prefixLenSum - 1)) {
+    } else {
+      assert commonPrefixLengths[sortedDim] < bytesPerDim;
       //estimate if storing the values with cardinality is cheaper than storing all values.
-      //+1 is the byte needed for storing the cardinality and -1 is the runLen compression.
-      out.writeByte((byte) -2);
-      if (numIndexDims != 1) {
-        writeActualBounds(out, commonPrefixLengths, count, packedValues);
-      }
-      BytesRef value = packedValues.apply(0);
-      System.arraycopy(value.bytes, value.offset, scratch1, 0, packedBytesLength);
-      int cardinality = 1;
-      for (int i = 1; i < count; i++) {
-        value = packedValues.apply(i);
-        if (Arrays.mismatch(value.bytes, value.offset, value.offset + value.length, scratch1, 0, packedBytesLength) != -1) {
-          out.writeVInt(cardinality);
-          for(int j = 0; j < numDataDims; j++) {
-            out.writeBytes(scratch1, j * bytesPerDim + commonPrefixLengths[j], bytesPerDim - commonPrefixLengths[j]);
-          }
-          System.arraycopy(value.bytes, value.offset, scratch1, 0, packedBytesLength);
-          cardinality = 1;
-        } else {
-          cardinality++;
+      //+1 is the byte needed for storing the cardinality
+      int lowCardinalityCost = leafCardinality * (packedBytesLength - prefixLenSum + 1);
+      int compressedByteOffset = sortedDim * bytesPerDim + commonPrefixLengths[sortedDim];
+      int highCardinalityCost;
+      if (count == leafCardinality) {
+        //all values in this block are different
+        highCardinalityCost = 0;
+      } else {
+        //compute cost of runLen compression
+        int numRunLens = 0;
+        for (int i = 0; i < count; ) {
+          // do run-length compression on the byte at compressedByteOffset
+          int runLen = runLen(packedValues, i, Math.min(i + 0xff, count), compressedByteOffset);
+          assert runLen <= 0xff;
+          numRunLens++;
+          i += runLen;
         }
+        //Add cost of runLen compression
+        highCardinalityCost = count * (packedBytesLength - prefixLenSum - 1) + 2 * numRunLens;
       }
-      out.writeVInt(cardinality);
-      for(int i = 0; i < numDataDims; i++) {
-        out.writeBytes(scratch1, i * bytesPerDim + commonPrefixLengths[i], bytesPerDim - commonPrefixLengths[i]);
-      }
-    } else {
-      assert commonPrefixLengths[sortedDim] < bytesPerDim;
-      out.writeByte((byte) sortedDim);
-      if (numIndexDims != 1) {
-        writeActualBounds(out, commonPrefixLengths, count, packedValues);
+      if (lowCardinalityCost <= highCardinalityCost) {
+        out.writeByte((byte) -2);
+        writeLowCardinalityLeafBlockPackedValues(out, commonPrefixLengths, count, packedValues);
+      } else {
+        out.writeByte((byte) sortedDim);
+        writeHighCardinalityLeafBlockPackedValues(out, commonPrefixLengths, count, sortedDim, packedValues, compressedByteOffset);
       }
-      int compressedByteOffset = sortedDim * bytesPerDim + commonPrefixLengths[sortedDim];
-      commonPrefixLengths[sortedDim]++;
-      for (int i = 0; i < count; ) {
-        // do run-length compression on the byte at compressedByteOffset
-        int runLen = runLen(packedValues, i, Math.min(i + 0xff, count), compressedByteOffset);
-        assert runLen <= 0xff;
-        BytesRef first = packedValues.apply(i);
-        byte prefixByte = first.bytes[first.offset + compressedByteOffset];
-        out.writeByte(prefixByte);
-        out.writeByte((byte) runLen);
-        writeLeafBlockPackedValuesRange(out, commonPrefixLengths, i, i + runLen, packedValues);
-        i += runLen;
-        assert i <= count;
+    }
+  }
+
+  private void writeLowCardinalityLeafBlockPackedValues(DataOutput out, int[] commonPrefixLengths, int count, IntFunction<BytesRef> packedValues) throws IOException {
+    if (numIndexDims != 1) {
+      writeActualBounds(out, commonPrefixLengths, count, packedValues);
+    }
+    BytesRef value = packedValues.apply(0);
+    System.arraycopy(value.bytes, value.offset, scratch1, 0, packedBytesLength);
+    int cardinality = 1;
+    for (int i = 1; i < count; i++) {
+      value = packedValues.apply(i);
+      if (Arrays.mismatch(value.bytes, value.offset, value.offset + value.length, scratch1, 0, packedBytesLength) != -1) {
+        out.writeVInt(cardinality);
+        for (int j = 0; j < numDataDims; j++) {
+          out.writeBytes(scratch1, j * bytesPerDim + commonPrefixLengths[j], bytesPerDim - commonPrefixLengths[j]);
+        }
+        System.arraycopy(value.bytes, value.offset, scratch1, 0, packedBytesLength);
+        cardinality = 1;
+      } else {
+        cardinality++;
       }
     }
+    out.writeVInt(cardinality);
+    for (int i = 0; i < numDataDims; i++) {
+      out.writeBytes(scratch1, i * bytesPerDim + commonPrefixLengths[i], bytesPerDim - commonPrefixLengths[i]);
+    }
+  }
+
+  private void writeHighCardinalityLeafBlockPackedValues(DataOutput out, int[] commonPrefixLengths, int count, int sortedDim, IntFunction<BytesRef> packedValues, int compressedByteOffset) throws IOException {
+    if (numIndexDims != 1) {
+      writeActualBounds(out, commonPrefixLengths, count, packedValues);
+    }
+    commonPrefixLengths[sortedDim]++;
+    for (int i = 0; i < count; ) {
+      // do run-length compression on the byte at compressedByteOffset
+      int runLen = runLen(packedValues, i, Math.min(i + 0xff, count), compressedByteOffset);
+      assert runLen <= 0xff;
+      BytesRef first = packedValues.apply(i);
+      byte prefixByte = first.bytes[first.offset + compressedByteOffset];
+      out.writeByte(prefixByte);
+      out.writeByte((byte) runLen);
+      writeLeafBlockPackedValuesRange(out, commonPrefixLengths, i, i + runLen, packedValues);
+      i += runLen;
+      assert i <= count;
+    }
   }
 
   private void writeActualBounds(DataOutput out, int[] commonPrefixLengths, int count, IntFunction<BytesRef> packedValues) throws IOException {

From 80ee25ddb7b32b67fb402eef620dcdf141cfe341 Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Sun, 23 Jun 2019 10:46:10 +0200
Subject: [PATCH 07/19] Add TODO

---
 lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index a925e852d926..7195167c1907 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -1316,6 +1316,7 @@ private void build(int nodeID, int leafNodeOffset,
       for (int i = from + 1; i < to; ++i) {
         reader.getValue(i, scratchBytesRef2);
         if (Arrays.mismatch(scratch1, 0, packedBytesLength, scratchBytesRef2.bytes, scratchBytesRef2.offset, scratchBytesRef2.offset + packedBytesLength) != -1) {
+          //TODO: Can we avoid this copy of the array?
           leafCardinality++;
           System.arraycopy(scratchBytesRef2.bytes, scratchBytesRef2.offset, scratch1, 0, packedBytesLength);
         }
@@ -1597,6 +1598,7 @@ private int computeCommonPrefixLength(HeapPointWriter heapPointWriter, byte[] co
       value =  heapPointWriter.getPackedValueSlice(i);
       packedValue = value.packedValue();
       if (Arrays.mismatch(scratch2, 0, packedBytesLength, packedValue.bytes, packedValue.offset, packedValue.offset + packedBytesLength) != -1) {
+        //TODO: Can we avoid this copy of the array?
         leafCardinality++;
         System.arraycopy(packedValue.bytes, packedValue.offset, scratch2, 0, packedBytesLength);
       }

From f904dfb614edeedd6102297b577653d2cd0fd0ab Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Sun, 23 Jun 2019 15:30:00 +0200
Subject: [PATCH 08/19] compute cardinality after sorting leaf and avoid
 copying of arrays

---
 .../org/apache/lucene/util/bkd/BKDWriter.java | 37 ++++++++++---------
 .../lucene/util/bkd/HeapPointWriter.java      | 13 +++++++
 2 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index 7195167c1907..2250729e7aa4 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -1308,18 +1308,11 @@ private void build(int nodeID, int leafNodeOffset,
       final int count = to - from;
       assert count <= maxPointsInLeafNode;
 
-      // Compute common prefixes and cardinality
+      // Compute common prefixes
       Arrays.fill(commonPrefixLengths, bytesPerDim);
       reader.getValue(from, scratchBytesRef1);
-      int leafCardinality = 1;
-      System.arraycopy(scratchBytesRef1.bytes, scratchBytesRef1.offset, scratch1, 0, packedBytesLength);
       for (int i = from + 1; i < to; ++i) {
         reader.getValue(i, scratchBytesRef2);
-        if (Arrays.mismatch(scratch1, 0, packedBytesLength, scratchBytesRef2.bytes, scratchBytesRef2.offset, scratchBytesRef2.offset + packedBytesLength) != -1) {
-          //TODO: Can we avoid this copy of the array?
-          leafCardinality++;
-          System.arraycopy(scratchBytesRef2.bytes, scratchBytesRef2.offset, scratch1, 0, packedBytesLength);
-        }
         for (int dim=0;dim<numDataDims;dim++) {
           final int offset = dim * bytesPerDim;
           int dimensionPrefixLength = commonPrefixLengths[dim];
@@ -1364,6 +1357,20 @@ private void build(int nodeID, int leafNodeOffset,
       MutablePointsReaderUtils.sortByDim(sortedDim, bytesPerDim, commonPrefixLengths,
           reader, from, to, scratchBytesRef1, scratchBytesRef2);
 
+      BytesRef comparator = scratchBytesRef1;
+      BytesRef collector = scratchBytesRef2;
+      reader.getValue(from, comparator);
+      int leafCardinality = 1;
+      for (int i = from + 1; i < to; ++i) {
+        reader.getValue(i, collector);
+        if (Arrays.mismatch(collector.bytes, collector.offset, collector.offset + packedBytesLength,
+            comparator.bytes, comparator.offset, comparator.offset + packedBytesLength) != -1) {
+          leafCardinality++;
+          BytesRef scratch = collector;
+          collector = comparator;
+          comparator = scratch;
+        }
+      }
       // Save the block file pointer:
       leafBlockFPs[nodeID - leafNodeOffset] = out.getFilePointer();
 
@@ -1464,7 +1471,7 @@ private void build(int nodeID, int leafNodeOffset,
       int from = Math.toIntExact(points.start);
       int to = Math.toIntExact(points.start + points.count);
       //we store common prefix on scratch1
-      int leafCardinality = computeCommonPrefixLength(heapSource, scratch1, from, to);
+      computeCommonPrefixLength(heapSource, scratch1, from, to);
 
       int sortedDim = 0;
       int sortedDimCardinality = Integer.MAX_VALUE;
@@ -1495,6 +1502,8 @@ private void build(int nodeID, int leafNodeOffset,
 
       // sort the chosen dimension
       radixSelector.heapRadixSort(heapSource, from, to, sortedDim, commonPrefixLengths[sortedDim]);
+      // compute cardinality
+      int leafCardinality = heapSource.computeCardinality(from ,to);
 
       // Save the block file pointer:
       leafBlockFPs[nodeID - leafNodeOffset] = out.getFilePointer();
@@ -1585,23 +1594,16 @@ assert valuesInOrderAndBounds(count, sortedDim, minPackedValue, maxPackedValue,
     }
   }
 
-  private int computeCommonPrefixLength(HeapPointWriter heapPointWriter, byte[] commonPrefix, int from, int to) {
+  private void computeCommonPrefixLength(HeapPointWriter heapPointWriter, byte[] commonPrefix, int from, int to) {
     Arrays.fill(commonPrefixLengths, bytesPerDim);
     PointValue value = heapPointWriter.getPackedValueSlice(from);
     BytesRef packedValue = value.packedValue();
     for (int dim = 0; dim < numDataDims; dim++) {
       System.arraycopy(packedValue.bytes, packedValue.offset + dim * bytesPerDim, commonPrefix, dim * bytesPerDim, bytesPerDim);
     }
-    System.arraycopy(packedValue.bytes, packedValue.offset, scratch2, 0, packedBytesLength);
-    int leafCardinality = 1;
     for (int i = from + 1; i < to; i++) {
       value =  heapPointWriter.getPackedValueSlice(i);
       packedValue = value.packedValue();
-      if (Arrays.mismatch(scratch2, 0, packedBytesLength, packedValue.bytes, packedValue.offset, packedValue.offset + packedBytesLength) != -1) {
-        //TODO: Can we avoid this copy of the array?
-        leafCardinality++;
-        System.arraycopy(packedValue.bytes, packedValue.offset, scratch2, 0, packedBytesLength);
-      }
       for (int dim = 0; dim < numDataDims; dim++) {
         if (commonPrefixLengths[dim] != 0) {
           int j = Arrays.mismatch(commonPrefix, dim * bytesPerDim, dim * bytesPerDim + commonPrefixLengths[dim], packedValue.bytes, packedValue.offset + dim * bytesPerDim, packedValue.offset + dim * bytesPerDim + commonPrefixLengths[dim]);
@@ -1611,7 +1613,6 @@ private int computeCommonPrefixLength(HeapPointWriter heapPointWriter, byte[] co
         }
       }
     }
-    return leafCardinality;
   }
 
   // only called from assert
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
index db30548e58be..9a63a1c4b20d 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
@@ -16,6 +16,8 @@
  */
 package org.apache.lucene.util.bkd;
 
+import java.util.Arrays;
+
 import org.apache.lucene.util.BytesRef;
 
 /**
@@ -93,6 +95,17 @@ public void swap(int i, int j) {
     System.arraycopy(scratch, 0, block, indexJ, packedBytesLength);
   }
 
+  public int computeCardinality(int from, int to) {
+    int leafCardinality = 1;
+    for (int i = from + 1; i < to; i++) {
+      if (Arrays.mismatch(block, i * packedBytesLength, (i + 1) * packedBytesLength,
+          block, (i -1) * packedBytesLength, i * packedBytesLength) != -1) {
+        leafCardinality++;
+      }
+    }
+    return leafCardinality;
+  }
+
   @Override
   public long count() {
     return nextWrite;

From 918d399e8c1a7dbc4474188af8d8d45fd4577ac7 Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Sun, 23 Jun 2019 21:09:37 +0200
Subject: [PATCH 09/19] clean up

---
 .../core/src/java/org/apache/lucene/util/bkd/BKDWriter.java | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index 2250729e7aa4..bca264e90808 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -1043,13 +1043,13 @@ private void writeLeafBlockPackedValues(DataOutput out, int[] commonPrefixLength
     } else {
       assert commonPrefixLengths[sortedDim] < bytesPerDim;
       //estimate if storing the values with cardinality is cheaper than storing all values.
-      //+1 is the byte needed for storing the cardinality
-      int lowCardinalityCost = leafCardinality * (packedBytesLength - prefixLenSum + 1);
       int compressedByteOffset = sortedDim * bytesPerDim + commonPrefixLengths[sortedDim];
       int highCardinalityCost;
+      int lowCardinalityCost;
       if (count == leafCardinality) {
         //all values in this block are different
         highCardinalityCost = 0;
+        lowCardinalityCost = 1;
       } else {
         //compute cost of runLen compression
         int numRunLens = 0;
@@ -1062,6 +1062,8 @@ private void writeLeafBlockPackedValues(DataOutput out, int[] commonPrefixLength
         }
         //Add cost of runLen compression
         highCardinalityCost = count * (packedBytesLength - prefixLenSum - 1) + 2 * numRunLens;
+        //+1 is the byte needed for storing the cardinality
+        lowCardinalityCost = leafCardinality * (packedBytesLength - prefixLenSum + 1);
       }
       if (lowCardinalityCost <= highCardinalityCost) {
         out.writeByte((byte) -2);

From 2f39af319631e7842d053390c1c10b121dfd3f96 Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Mon, 24 Jun 2019 06:49:22 +0200
Subject: [PATCH 10/19] clean up

---
 lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index bca264e90808..dee124c4ce89 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -1084,7 +1084,7 @@ private void writeLowCardinalityLeafBlockPackedValues(DataOutput out, int[] comm
     int cardinality = 1;
     for (int i = 1; i < count; i++) {
       value = packedValues.apply(i);
-      if (Arrays.mismatch(value.bytes, value.offset, value.offset + value.length, scratch1, 0, packedBytesLength) != -1) {
+      if (Arrays.mismatch(value.bytes, value.offset, value.offset + packedBytesLength, scratch1, 0, packedBytesLength) != -1) {
         out.writeVInt(cardinality);
         for (int j = 0; j < numDataDims; j++) {
           out.writeBytes(scratch1, j * bytesPerDim + commonPrefixLengths[j], bytesPerDim - commonPrefixLengths[j]);

From 6eaf5a1f4be070b9c8c32df8b18ea055473058ed Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Mon, 24 Jun 2019 07:14:37 +0200
Subject: [PATCH 11/19] leverage the commonPrefixLength info for mismatch

---
 .../org/apache/lucene/util/bkd/BKDWriter.java | 37 +++++++++++--------
 .../lucene/util/bkd/HeapPointWriter.java      | 12 ++++--
 2 files changed, 30 insertions(+), 19 deletions(-)

diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index dee124c4ce89..ffbc636e8998 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -1084,15 +1084,19 @@ private void writeLowCardinalityLeafBlockPackedValues(DataOutput out, int[] comm
     int cardinality = 1;
     for (int i = 1; i < count; i++) {
       value = packedValues.apply(i);
-      if (Arrays.mismatch(value.bytes, value.offset, value.offset + packedBytesLength, scratch1, 0, packedBytesLength) != -1) {
-        out.writeVInt(cardinality);
-        for (int j = 0; j < numDataDims; j++) {
-          out.writeBytes(scratch1, j * bytesPerDim + commonPrefixLengths[j], bytesPerDim - commonPrefixLengths[j]);
+      for(int dim =0; dim < numDataDims; dim++) {
+        if (Arrays.mismatch(value.bytes, value.offset + dim * bytesPerDim + commonPrefixLengths[dim], value.offset + dim * bytesPerDim + bytesPerDim,
+            scratch1, dim * bytesPerDim + commonPrefixLengths[dim], + dim * bytesPerDim + bytesPerDim) != -1) {
+          out.writeVInt(cardinality);
+          for (int j = 0; j < numDataDims; j++) {
+            out.writeBytes(scratch1, j * bytesPerDim + commonPrefixLengths[j], bytesPerDim - commonPrefixLengths[j]);
+          }
+          System.arraycopy(value.bytes, value.offset, scratch1, 0, packedBytesLength);
+          cardinality = 1;
+          break;
+        } else if (dim == numDataDims - 1){
+          cardinality++;
         }
-        System.arraycopy(value.bytes, value.offset, scratch1, 0, packedBytesLength);
-        cardinality = 1;
-      } else {
-        cardinality++;
       }
     }
     out.writeVInt(cardinality);
@@ -1365,12 +1369,15 @@ private void build(int nodeID, int leafNodeOffset,
       int leafCardinality = 1;
       for (int i = from + 1; i < to; ++i) {
         reader.getValue(i, collector);
-        if (Arrays.mismatch(collector.bytes, collector.offset, collector.offset + packedBytesLength,
-            comparator.bytes, comparator.offset, comparator.offset + packedBytesLength) != -1) {
-          leafCardinality++;
-          BytesRef scratch = collector;
-          collector = comparator;
-          comparator = scratch;
+        for (int dim =0; dim < numDataDims; dim++) {
+          if (Arrays.mismatch(collector.bytes, collector.offset + dim * bytesPerDim + commonPrefixLengths[dim], collector.offset +  dim * bytesPerDim + bytesPerDim,
+              comparator.bytes, comparator.offset + dim * bytesPerDim + commonPrefixLengths[dim], comparator.offset + +  dim * bytesPerDim + bytesPerDim) != -1) {
+            leafCardinality++;
+            BytesRef scratch = collector;
+            collector = comparator;
+            comparator = scratch;
+            break;
+          }
         }
       }
       // Save the block file pointer:
@@ -1505,7 +1512,7 @@ private void build(int nodeID, int leafNodeOffset,
       // sort the chosen dimension
       radixSelector.heapRadixSort(heapSource, from, to, sortedDim, commonPrefixLengths[sortedDim]);
       // compute cardinality
-      int leafCardinality = heapSource.computeCardinality(from ,to);
+      int leafCardinality = heapSource.computeCardinality(from ,to, numDataDims, bytesPerDim, commonPrefixLengths);
 
       // Save the block file pointer:
       leafBlockFPs[nodeID - leafNodeOffset] = out.getFilePointer();
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
index 9a63a1c4b20d..0a25a9a912af 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
@@ -95,12 +95,16 @@ public void swap(int i, int j) {
     System.arraycopy(scratch, 0, block, indexJ, packedBytesLength);
   }
 
-  public int computeCardinality(int from, int to) {
+  public int computeCardinality(int from, int to, int numDataDims, int bytesPerDim, int[] commonPrefixLengths) {
+    assert packedBytesLength == numDataDims * bytesPerDim;
     int leafCardinality = 1;
     for (int i = from + 1; i < to; i++) {
-      if (Arrays.mismatch(block, i * packedBytesLength, (i + 1) * packedBytesLength,
-          block, (i -1) * packedBytesLength, i * packedBytesLength) != -1) {
-        leafCardinality++;
+      for (int dim =0; dim < numDataDims; dim++) {
+        if (Arrays.mismatch(block, i * packedBytesLength + dim * bytesPerDim + commonPrefixLengths[dim], i * packedBytesLength + dim * bytesPerDim + bytesPerDim,
+            block, (i - 1) * packedBytesLength  + dim * bytesPerDim + commonPrefixLengths[dim], (i -1) * packedBytesLength + dim * bytesPerDim + bytesPerDim) != -1) {
+          leafCardinality++;
+          break;
+        }
       }
     }
     return leafCardinality;

From 13f1f51815672c09b2c2e2f551533e950a150afa Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Mon, 24 Jun 2019 07:20:25 +0200
Subject: [PATCH 12/19] clean up

---
 .../java/org/apache/lucene/util/bkd/BKDWriter.java    | 11 +++++++----
 .../org/apache/lucene/util/bkd/HeapPointWriter.java   |  6 ++++--
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index ffbc636e8998..e472d2c05b79 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -1085,8 +1085,9 @@ private void writeLowCardinalityLeafBlockPackedValues(DataOutput out, int[] comm
     for (int i = 1; i < count; i++) {
       value = packedValues.apply(i);
       for(int dim =0; dim < numDataDims; dim++) {
-        if (Arrays.mismatch(value.bytes, value.offset + dim * bytesPerDim + commonPrefixLengths[dim], value.offset + dim * bytesPerDim + bytesPerDim,
-            scratch1, dim * bytesPerDim + commonPrefixLengths[dim], + dim * bytesPerDim + bytesPerDim) != -1) {
+        final int start = dim * bytesPerDim + commonPrefixLengths[dim];
+        final int end = dim * bytesPerDim + bytesPerDim;
+        if (Arrays.mismatch(value.bytes, value.offset + start, value.offset + end, scratch1, start, end) != -1) {
           out.writeVInt(cardinality);
           for (int j = 0; j < numDataDims; j++) {
             out.writeBytes(scratch1, j * bytesPerDim + commonPrefixLengths[j], bytesPerDim - commonPrefixLengths[j]);
@@ -1370,8 +1371,10 @@ private void build(int nodeID, int leafNodeOffset,
       for (int i = from + 1; i < to; ++i) {
         reader.getValue(i, collector);
         for (int dim =0; dim < numDataDims; dim++) {
-          if (Arrays.mismatch(collector.bytes, collector.offset + dim * bytesPerDim + commonPrefixLengths[dim], collector.offset +  dim * bytesPerDim + bytesPerDim,
-              comparator.bytes, comparator.offset + dim * bytesPerDim + commonPrefixLengths[dim], comparator.offset + +  dim * bytesPerDim + bytesPerDim) != -1) {
+          final int start = dim * bytesPerDim + commonPrefixLengths[dim];
+          final int end = dim * bytesPerDim + bytesPerDim;
+          if (Arrays.mismatch(collector.bytes, collector.offset + start, collector.offset + end,
+              comparator.bytes, comparator.offset + start, comparator.offset + end) != -1) {
             leafCardinality++;
             BytesRef scratch = collector;
             collector = comparator;
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
index 0a25a9a912af..16b07218ed45 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
@@ -100,8 +100,10 @@ public int computeCardinality(int from, int to, int numDataDims, int bytesPerDim
     int leafCardinality = 1;
     for (int i = from + 1; i < to; i++) {
       for (int dim =0; dim < numDataDims; dim++) {
-        if (Arrays.mismatch(block, i * packedBytesLength + dim * bytesPerDim + commonPrefixLengths[dim], i * packedBytesLength + dim * bytesPerDim + bytesPerDim,
-            block, (i - 1) * packedBytesLength  + dim * bytesPerDim + commonPrefixLengths[dim], (i -1) * packedBytesLength + dim * bytesPerDim + bytesPerDim) != -1) {
+        final int start = dim * bytesPerDim + commonPrefixLengths[dim];
+        final int end = dim * bytesPerDim + bytesPerDim;
+        if (Arrays.mismatch(block, i * packedBytesLength + start, i * packedBytesLength + end,
+            block, (i - 1) * packedBytesLength  + start, (i -1) * packedBytesLength + end) != -1) {
           leafCardinality++;
           break;
         }

From c8103c7a55c9ad663456eb2b45b0c1cca76e389e Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Mon, 24 Jun 2019 10:39:20 +0200
Subject: [PATCH 13/19] Address review comments, mostly formatting

---
 .../org/apache/lucene/util/bkd/BKDReader.java | 22 +++++++++----------
 .../org/apache/lucene/util/bkd/BKDWriter.java | 12 +++++-----
 .../lucene/util/bkd/HeapPointWriter.java      |  2 +-
 3 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
index c8230c6a56b7..04b213b9dbb0 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
@@ -456,8 +456,7 @@ void visitDocValuesNoCardinality(int[] commonPrefixLengths, byte[] scratchDataPa
       byte[] minPackedValue = scratchMinIndexPackedValue;
       System.arraycopy(scratchDataPackedValue, 0, minPackedValue, 0, packedIndexBytesLength);
       byte[] maxPackedValue = scratchMaxIndexPackedValue;
-      //Copy common prefixes before reading adjusted
-      // box
+      // Copy common prefixes before reading adjusted box
       System.arraycopy(minPackedValue, 0, maxPackedValue, 0, packedIndexBytesLength);
       readMinMax(commonPrefixLengths, minPackedValue, maxPackedValue, in);
 
@@ -499,16 +498,15 @@ void visitDocValuesWithCardinality(int[] commonPrefixLengths, byte[] scratchData
     readCommonPrefixes(commonPrefixLengths, scratchDataPackedValue, in);
     int compressedDim = readCompressedDim(in);
     if (compressedDim == -1) {
-      //all values are the same
+      // all values are the same
       visitor.grow(count);
       visitUniqueRawDocValues(scratchDataPackedValue, docIDs, count, visitor);
     } else {
-      if (numIndexDims != 1 && version >= BKDWriter.VERSION_LEAF_STORES_BOUNDS) {
+      if (numIndexDims != 1) {
         byte[] minPackedValue = scratchMinIndexPackedValue;
         System.arraycopy(scratchDataPackedValue, 0, minPackedValue, 0, packedIndexBytesLength);
         byte[] maxPackedValue = scratchMaxIndexPackedValue;
-        //Copy common prefixes before reading adjusted
-        // box
+        // Copy common prefixes before reading adjusted box
         System.arraycopy(minPackedValue, 0, maxPackedValue, 0, packedIndexBytesLength);
         readMinMax(commonPrefixLengths, minPackedValue, maxPackedValue, in);
 
@@ -534,10 +532,10 @@ void visitDocValuesWithCardinality(int[] commonPrefixLengths, byte[] scratchData
         visitor.grow(count);
       }
       if (compressedDim == -2) {
-        //low cardinality values
+        // low cardinality values
         visitSparseRawDocValues(commonPrefixLengths, scratchDataPackedValue, in, docIDs, count, visitor);
       } else {
-        //high cardinality
+        // high cardinality
         visitCompressedDocValues(commonPrefixLengths, scratchDataPackedValue, in, docIDs, count, visitor, compressedDim);
       }
     }
@@ -551,12 +549,12 @@ private void readMinMax(int[] commonPrefixLengths, byte[] minPackedValue, byte[]
     }
   }
 
-  // Just read suffixes for every dimension
+  // read cardinality and point
   private void visitSparseRawDocValues(int[] commonPrefixLengths, byte[] scratchPackedValue, IndexInput in, int[] docIDs, int count, IntersectVisitor visitor) throws IOException {
     int i;
     for (i = 0; i < count;) {
       int length = in.readVInt();
-      for(int dim=0;dim<numDataDims;dim++) {
+      for(int dim = 0; dim < numDataDims; dim++) {
         int prefix = commonPrefixLengths[dim];
         in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix);
       }
@@ -570,7 +568,7 @@ private void visitSparseRawDocValues(int[] commonPrefixLengths, byte[] scratchPa
     }
   }
 
-  // Just read suffixes for every dimension
+  // point is under commonPrefix
   private void visitUniqueRawDocValues(byte[] scratchPackedValue, int[] docIDs, int count, IntersectVisitor visitor) throws IOException {
     for (int i = 0; i < count; i++) {
       visitor.visit(docIDs[i], scratchPackedValue);
@@ -587,7 +585,7 @@ private void visitCompressedDocValues(int[] commonPrefixLengths, byte[] scratchP
       scratchPackedValue[compressedByteOffset] = in.readByte();
       final int runLen = Byte.toUnsignedInt(in.readByte());
       for (int j = 0; j < runLen; ++j) {
-        for(int dim=0;dim<numDataDims;dim++) {
+        for(int dim = 0; dim < numDataDims; dim++) {
           int prefix = commonPrefixLengths[dim];
           in.readBytes(scratchPackedValue, dim*bytesPerDim + prefix, bytesPerDim - prefix);
         }
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index e472d2c05b79..c2b9545c7d8b 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -80,7 +80,7 @@ public class BKDWriter implements Closeable {
   //public static final int VERSION_CURRENT = VERSION_START;
   public static final int VERSION_LEAF_STORES_BOUNDS = 5;
   public static final int VERSION_SELECTIVE_INDEXING = 6;
-  public static final int VERSION_LOW_CARDINALITY_LEAVES= 7;
+  public static final int VERSION_LOW_CARDINALITY_LEAVES = 7;
   public static final int VERSION_CURRENT = VERSION_LOW_CARDINALITY_LEAVES;
 
   /** How many bytes each docs takes in the fixed-width offline format */
@@ -1042,16 +1042,16 @@ private void writeLeafBlockPackedValues(DataOutput out, int[] commonPrefixLength
       out.writeByte((byte) -1);
     } else {
       assert commonPrefixLengths[sortedDim] < bytesPerDim;
-      //estimate if storing the values with cardinality is cheaper than storing all values.
+      // estimate if storing the values with cardinality is cheaper than storing all values.
       int compressedByteOffset = sortedDim * bytesPerDim + commonPrefixLengths[sortedDim];
       int highCardinalityCost;
       int lowCardinalityCost;
       if (count == leafCardinality) {
-        //all values in this block are different
+        // all values in this block are different
         highCardinalityCost = 0;
         lowCardinalityCost = 1;
       } else {
-        //compute cost of runLen compression
+        // compute cost of runLen compression
         int numRunLens = 0;
         for (int i = 0; i < count; ) {
           // do run-length compression on the byte at compressedByteOffset
@@ -1060,9 +1060,9 @@ private void writeLeafBlockPackedValues(DataOutput out, int[] commonPrefixLength
           numRunLens++;
           i += runLen;
         }
-        //Add cost of runLen compression
+        // Add cost of runLen compression
         highCardinalityCost = count * (packedBytesLength - prefixLenSum - 1) + 2 * numRunLens;
-        //+1 is the byte needed for storing the cardinality
+        // +1 is the byte needed for storing the cardinality
         lowCardinalityCost = leafCardinality * (packedBytesLength - prefixLenSum + 1);
       }
       if (lowCardinalityCost <= highCardinalityCost) {
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
index 16b07218ed45..a86ea50592d8 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
@@ -99,7 +99,7 @@ public int computeCardinality(int from, int to, int numDataDims, int bytesPerDim
     assert packedBytesLength == numDataDims * bytesPerDim;
     int leafCardinality = 1;
     for (int i = from + 1; i < to; i++) {
-      for (int dim =0; dim < numDataDims; dim++) {
+      for (int dim = 0; dim < numDataDims; dim++) {
         final int start = dim * bytesPerDim + commonPrefixLengths[dim];
         final int end = dim * bytesPerDim + bytesPerDim;
         if (Arrays.mismatch(block, i * packedBytesLength + start, i * packedBytesLength + end,

From 5054ff4be52c87ca1c540b1da4badc644cd89eff Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Mon, 24 Jun 2019 15:24:42 +0200
Subject: [PATCH 14/19] formatting

---
 lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
index 04b213b9dbb0..ffabd6a6eeff 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDReader.java
@@ -561,7 +561,7 @@ private void visitSparseRawDocValues(int[] commonPrefixLengths, byte[] scratchPa
       for (int j = i; j < i + length; j++) {
         visitor.visit(docIDs[j], scratchPackedValue);
       }
-      i+= length;
+      i += length;
     }
     if (i != count) {
       throw new CorruptIndexException("Sub blocks do not add up to the expected count: " + count + " != " + i, in);

From 885ea933f74c0e2288e7fae9ee4f20b248103e41 Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Mon, 24 Jun 2019 15:27:46 +0200
Subject: [PATCH 15/19] formatting

---
 lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
index c2b9545c7d8b..c65d56adeb86 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/BKDWriter.java
@@ -1084,7 +1084,7 @@ private void writeLowCardinalityLeafBlockPackedValues(DataOutput out, int[] comm
     int cardinality = 1;
     for (int i = 1; i < count; i++) {
       value = packedValues.apply(i);
-      for(int dim =0; dim < numDataDims; dim++) {
+      for(int dim = 0; dim < numDataDims; dim++) {
         final int start = dim * bytesPerDim + commonPrefixLengths[dim];
         final int end = dim * bytesPerDim + bytesPerDim;
         if (Arrays.mismatch(value.bytes, value.offset + start, value.offset + end, scratch1, start, end) != -1) {

From a1471fec7dbd877d4f8e7a977992cfab152ee572 Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Mon, 24 Jun 2019 15:29:34 +0200
Subject: [PATCH 16/19] formatting

---
 .../src/java/org/apache/lucene/util/bkd/HeapPointWriter.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
index a86ea50592d8..306ec5c46ee9 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
@@ -103,7 +103,7 @@ public int computeCardinality(int from, int to, int numDataDims, int bytesPerDim
         final int start = dim * bytesPerDim + commonPrefixLengths[dim];
         final int end = dim * bytesPerDim + bytesPerDim;
         if (Arrays.mismatch(block, i * packedBytesLength + start, i * packedBytesLength + end,
-            block, (i - 1) * packedBytesLength  + start, (i -1) * packedBytesLength + end) != -1) {
+            block, (i - 1) * packedBytesLength  + start, (i - 1) * packedBytesLength + end) != -1) {
           leafCardinality++;
           break;
         }

From 9f453d749efbefa9406fff0e877e353506517e36 Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Tue, 25 Jun 2019 06:55:20 +0200
Subject: [PATCH 17/19] Add test that triggers low cardinality leaves

---
 .../org/apache/lucene/util/bkd/TestBKD.java   | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
index ce76ca47072d..72cc0934d7b3 100644
--- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
+++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
@@ -614,6 +614,29 @@ public void testOneDimTwoValues() throws Exception {
     verify(docValues, null, numDataDims, numIndexDims, numBytesPerDim);
   }
 
+  //this should trigger low cardinality leaves
+  public void testRandomFewDifferentValues() throws Exception {
+    int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
+    int numIndexDims = TestUtil.nextInt(random(), 1, 8);
+    int numDataDims = TestUtil.nextInt(random(), numIndexDims, 8);
+
+    int numDocs = atLeast(10000);
+    int cardinality = TestUtil.nextInt(random(), 2, 100);
+    byte[][][] values = new byte[cardinality][numDataDims][numBytesPerDim];
+    for (int i =0; i < cardinality; i++) {
+      for (int j =0; j < numDataDims; j ++) {
+        random().nextBytes(values[i][j]);
+      }
+    }
+
+    byte[][][] docValues = new byte[numDocs][][];
+    for(int docID=0;docID<numDocs;docID++) {
+      docValues[docID] = values[random().nextInt(cardinality)];
+    }
+
+    verify(docValues, null, numDataDims, numIndexDims, numBytesPerDim);
+  }
+
   public void testMultiValued() throws Exception {
     int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
     int numDataDims = TestUtil.nextInt(random(), 1, 5);

From 30f36ce66e294c537fa17685245d44f5009b88d1 Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Wed, 26 Jun 2019 10:01:43 +0200
Subject: [PATCH 18/19] formatting

---
 .../core/src/test/org/apache/lucene/util/bkd/TestBKD.java | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
index 72cc0934d7b3..15ba4f9408a6 100644
--- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
+++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java
@@ -614,7 +614,7 @@ public void testOneDimTwoValues() throws Exception {
     verify(docValues, null, numDataDims, numIndexDims, numBytesPerDim);
   }
 
-  //this should trigger low cardinality leaves
+  // this should trigger low cardinality leaves
   public void testRandomFewDifferentValues() throws Exception {
     int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
     int numIndexDims = TestUtil.nextInt(random(), 1, 8);
@@ -623,14 +623,14 @@ public void testRandomFewDifferentValues() throws Exception {
     int numDocs = atLeast(10000);
     int cardinality = TestUtil.nextInt(random(), 2, 100);
     byte[][][] values = new byte[cardinality][numDataDims][numBytesPerDim];
-    for (int i =0; i < cardinality; i++) {
-      for (int j =0; j < numDataDims; j ++) {
+    for (int i = 0; i < cardinality; i++) {
+      for (int j = 0; j < numDataDims; j++) {
         random().nextBytes(values[i][j]);
       }
     }
 
     byte[][][] docValues = new byte[numDocs][][];
-    for(int docID=0;docID<numDocs;docID++) {
+    for(int docID = 0; docID < numDocs; docID++) {
       docValues[docID] = values[random().nextInt(cardinality)];
     }
 

From cc08c5ff2fb6846dd51ab8256f3e82b277758364 Mon Sep 17 00:00:00 2001
From: iverase <ivera@apache.org>
Date: Wed, 26 Jun 2019 10:05:52 +0200
Subject: [PATCH 19/19] Add entry in Changes.txt

---
 lucene/CHANGES.txt | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index ba1fb2a20819..a8734a3d413f 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -128,6 +128,10 @@ Optimizations
   Now caller threads execute at least one search on an index even if there is
   an executor provided to minimize thread context switching. (Simon Willnauer)
 
+* LUCENE-8868: New storing strategy for BKD tree leaves with low cardinality.
+  It stores the distinct values once with the cardinality value reducing the
+  storage cost.
+
 Test Framework
 
 * LUCENE-8825: CheckHits now display the shard index in case of mismatch