From d3ed53e6392e9fd16a0a5b557170036ddc3f23bd Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 3 Nov 2025 11:51:05 +0100 Subject: [PATCH 1/7] NumericComparator: immediately check whether a segment is comparative with the recorded bottom. When construction a new CompetitiveDISIBuilder, then check whether global min/max points or global min/max doc values skipper are comparative with the bottom. If so, then update competitiveIterator with an empty iterator, because no documents will have a value that is competitive with the current recorded bottom in the current segment. Doing this at CompetitiveDISIBuilder construction is cheap and allows to immediately prune, instead of waiting until doUpdateCompetitiveIterator(...) is invoked. --- .../search/comparators/NumericComparator.java | 43 +++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java index 37efe32ec7d3..358e23d75205 100644 --- a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java +++ b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java @@ -320,7 +320,8 @@ private class PointsCompetitiveDISIBuilder extends CompetitiveDISIBuilder { // helps to be conservative about increasing the sampling interval private int tryUpdateFailCount = 0; - public PointsCompetitiveDISIBuilder(PointValues pointValues, NumericLeafComparator comparator) { + PointsCompetitiveDISIBuilder(PointValues pointValues, NumericLeafComparator comparator) + throws IOException { super(comparator); LeafReaderContext context = comparator.context; FieldInfo info = context.reader().getFieldInfos().fieldInfo(field); @@ -344,6 +345,7 @@ public PointsCompetitiveDISIBuilder(PointValues pointValues, NumericLeafComparat + bytesCount); } this.pointValues = pointValues; + postInitializeCompetitiveIterator(); } @Override @@ -364,6 +366,25 @@ int docCount() { return pointValues.getDocCount(); } + /** + * If queue is full and global min/max point values are not competitive with bottom then set an + * empty iterator as competitive iterator. + * + * @throws IOException i/o exception while fetching min and max values from point values + */ + void postInitializeCompetitiveIterator() throws IOException { + if (queueFull) { + long bottom = leafComparator.bottomAsComparableLong(); + long minValue = sortableBytesToLong(pointValues.getMinPackedValue()); + long maxValue = sortableBytesToLong(pointValues.getMaxPackedValue()); + if (reverse == false && bottom < minValue) { + competitiveIterator.update(DocIdSetIterator.empty()); + } else if (reverse && bottom > maxValue) { + competitiveIterator.update(DocIdSetIterator.empty()); + } + } + } + @Override void doUpdateCompetitiveIterator() throws IOException { DocIdSetBuilder result = new DocIdSetBuilder(maxDoc); @@ -478,8 +499,8 @@ private class DVSkipperCompetitiveDISIBuilder extends CompetitiveDISIBuilder { private final DocValuesSkipper skipper; private final TwoPhaseIterator innerTwoPhase; - public DVSkipperCompetitiveDISIBuilder( - DocValuesSkipper skipper, NumericLeafComparator leafComparator) throws IOException { + DVSkipperCompetitiveDISIBuilder(DocValuesSkipper skipper, NumericLeafComparator leafComparator) + throws IOException { super(leafComparator); this.skipper = skipper; NumericDocValues docValues = @@ -497,6 +518,7 @@ public float matchCost() { return 2; // 2 comparisons } }; + postInitializeCompetitiveIterator(); } @Override @@ -504,6 +526,21 @@ int docCount() { return skipper.docCount(); } + /** + * If queue is full and global min/max skipper are not competitive with bottom then set an empty + * iterator as competitive iterator. + */ + void postInitializeCompetitiveIterator() { + if (queueFull) { + long bottom = leafComparator.bottomAsComparableLong(); + if (reverse == false && bottom < skipper.minValue()) { + competitiveIterator.update(DocIdSetIterator.empty()); + } else if (reverse && bottom > skipper.maxValue()) { + competitiveIterator.update(DocIdSetIterator.empty()); + } + } + } + @Override void doUpdateCompetitiveIterator() { TwoPhaseIterator twoPhaseIterator = From a7697630c3616ecb228c80ecf1a64808bf942851 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 3 Nov 2025 12:12:41 +0100 Subject: [PATCH 2/7] added changelog entry --- lucene/CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index b88ef1dac63e..0742b8d96fa6 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -213,6 +213,8 @@ Optimizations * GITHUB#14963: Bypass HNSW graph building for tiny segments. (Shubham Chaudhary, Ben Trent) +* GITHUB#15397: NumericComparator: immediately check whether a segment is competitive with the recorded bottom (Martijn van Groningen) + Bug Fixes --------------------- * GITHUB#14161: PointInSetQuery's constructor now throws IllegalArgumentException From 6583d5fedcc8136f33ac7481edf30d37ec5b4251 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 3 Nov 2025 15:31:50 +0100 Subject: [PATCH 3/7] added unit test that verifies optimization kicks in --- .../comparators/TestNumericComparator.java | 159 ++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java diff --git a/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java b/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java new file mode 100644 index 000000000000..b23c2634caa2 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.search.comparators; + +import java.io.IOException; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.SortField; +import org.apache.lucene.tests.util.LuceneTestCase; + +public class TestNumericComparator extends LuceneTestCase { + + public void testEmptyCompetitiveIteratorOptimization() throws Exception { + final int numDocs = atLeast(1000); + try (var dir = newDirectory()) { + try (var writer = + new IndexWriter(dir, new IndexWriterConfig().setMergePolicy(newLogMergePolicy()))) { + for (int i = 0; i < numDocs; i++) { + var doc = new Document(); + + doc.add(NumericDocValuesField.indexedField("long_field_1", i)); + doc.add(new LongPoint("long_field_2", i)); + doc.add(new NumericDocValuesField("long_field_2", i)); + + doc.add(NumericDocValuesField.indexedField("int_field_1", i)); + doc.add(new IntPoint("int_field_2", i)); + doc.add(new NumericDocValuesField("int_field_2", i)); + + doc.add(NumericDocValuesField.indexedField("float_field_1", i)); + doc.add(new FloatPoint("float_field_2", i)); + doc.add(new NumericDocValuesField("float_field_2", i)); + + doc.add(NumericDocValuesField.indexedField("double_field_1", i)); + doc.add(new DoublePoint("double_field_2", i)); + doc.add(new NumericDocValuesField("double_field_2", i)); + + writer.addDocument(doc); + writer.forceMerge(1); + try (var reader = DirectoryReader.open(writer)) { + assertEquals(1, reader.leaves().size()); + var leafContext = reader.leaves().get(0); + + // long field 1 ascending: + assertLongField("long_field_1", false, -1, leafContext); + // long field 1 descending: + assertLongField("long_field_1", true, numDocs + 1, leafContext); + // long field 2 ascending: + assertLongField("long_field_2", false, -1, leafContext); + // long field 2 descending: + assertLongField("long_field_2", true, numDocs + 1, leafContext); + + // int field 1 ascending: + assertIntField("int_field_1", false, -1, leafContext); + // int field 1 descending: + assertIntField("int_field_1", true, numDocs + 1, leafContext); + // int field 2 ascending: + assertIntField("int_field_2", false, -1, leafContext); + // int field 2 descending: + assertIntField("int_field_2", true, numDocs + 1, leafContext); + + // float field 1 ascending: + assertFloatField("float_field_1", false, -1, leafContext); + // float field 1 descending: + assertFloatField("float_field_1", true, numDocs + 1, leafContext); + // float field 2 ascending: + assertFloatField("float_field_2", false, -1, leafContext); + // float field 2 descending: + assertFloatField("float_field_2", true, numDocs + 1, leafContext); + + // double field 1 ascending: + assertDoubleField("double_field_1", false, -1, leafContext); + // double field 1 descending: + assertDoubleField("double_field_1", true, numDocs + 1, leafContext); + // double field 2 ascending: + assertDoubleField("double_field_2", false, -1, leafContext); + // double field 2 descending: + assertDoubleField("double_field_2", true, numDocs + 1, leafContext); + } + } + } + } + } + + private static void assertLongField( + String fieldName, boolean reverse, int bottom, LeafReaderContext leafContext) + throws IOException { + var comparator1 = + (LongComparator) + new SortField(fieldName, SortField.Type.LONG, reverse) + .getComparator(1, Pruning.GREATER_THAN_OR_EQUAL_TO); + comparator1.queueFull = true; + comparator1.bottom = bottom; + var leafComparator = comparator1.getLeafComparator(leafContext); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, leafComparator.competitiveIterator().nextDoc()); + } + + private static void assertIntField( + String fieldName, boolean reverse, int bottom, LeafReaderContext leafContext) + throws IOException { + var comparator1 = + (IntComparator) + new SortField(fieldName, SortField.Type.INT, reverse) + .getComparator(1, Pruning.GREATER_THAN_OR_EQUAL_TO); + comparator1.queueFull = true; + comparator1.bottom = bottom; + var leafComparator = comparator1.getLeafComparator(leafContext); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, leafComparator.competitiveIterator().nextDoc()); + } + + private static void assertFloatField( + String fieldName, boolean reverse, int bottom, LeafReaderContext leafContext) + throws IOException { + var comparator1 = + (FloatComparator) + new SortField(fieldName, SortField.Type.FLOAT, reverse) + .getComparator(1, Pruning.GREATER_THAN_OR_EQUAL_TO); + comparator1.queueFull = true; + comparator1.bottom = bottom; + var leafComparator = comparator1.getLeafComparator(leafContext); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, leafComparator.competitiveIterator().nextDoc()); + } + + private static void assertDoubleField( + String fieldName, boolean reverse, int bottom, LeafReaderContext leafContext) + throws IOException { + var comparator1 = + (DoubleComparator) + new SortField(fieldName, SortField.Type.DOUBLE, reverse) + .getComparator(1, Pruning.GREATER_THAN_OR_EQUAL_TO); + comparator1.queueFull = true; + comparator1.bottom = bottom; + var leafComparator = comparator1.getLeafComparator(leafContext); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, leafComparator.competitiveIterator().nextDoc()); + } +} From 2be09bb091da02c866d1f1dd0e99d4ffbc2d79a4 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 4 Nov 2025 11:35:51 +0100 Subject: [PATCH 4/7] test: open reader after all docs have been indexed. --- .../comparators/TestNumericComparator.java | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java b/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java index b23c2634caa2..8846a9df5f9d 100644 --- a/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java +++ b/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java @@ -60,46 +60,46 @@ public void testEmptyCompetitiveIteratorOptimization() throws Exception { writer.addDocument(doc); writer.forceMerge(1); - try (var reader = DirectoryReader.open(writer)) { - assertEquals(1, reader.leaves().size()); - var leafContext = reader.leaves().get(0); - - // long field 1 ascending: - assertLongField("long_field_1", false, -1, leafContext); - // long field 1 descending: - assertLongField("long_field_1", true, numDocs + 1, leafContext); - // long field 2 ascending: - assertLongField("long_field_2", false, -1, leafContext); - // long field 2 descending: - assertLongField("long_field_2", true, numDocs + 1, leafContext); - - // int field 1 ascending: - assertIntField("int_field_1", false, -1, leafContext); - // int field 1 descending: - assertIntField("int_field_1", true, numDocs + 1, leafContext); - // int field 2 ascending: - assertIntField("int_field_2", false, -1, leafContext); - // int field 2 descending: - assertIntField("int_field_2", true, numDocs + 1, leafContext); - - // float field 1 ascending: - assertFloatField("float_field_1", false, -1, leafContext); - // float field 1 descending: - assertFloatField("float_field_1", true, numDocs + 1, leafContext); - // float field 2 ascending: - assertFloatField("float_field_2", false, -1, leafContext); - // float field 2 descending: - assertFloatField("float_field_2", true, numDocs + 1, leafContext); - - // double field 1 ascending: - assertDoubleField("double_field_1", false, -1, leafContext); - // double field 1 descending: - assertDoubleField("double_field_1", true, numDocs + 1, leafContext); - // double field 2 ascending: - assertDoubleField("double_field_2", false, -1, leafContext); - // double field 2 descending: - assertDoubleField("double_field_2", true, numDocs + 1, leafContext); - } + } + try (var reader = DirectoryReader.open(writer)) { + assertEquals(1, reader.leaves().size()); + var leafContext = reader.leaves().get(0); + + // long field 1 ascending: + assertLongField("long_field_1", false, -1, leafContext); + // long field 1 descending: + assertLongField("long_field_1", true, numDocs + 1, leafContext); + // long field 2 ascending: + assertLongField("long_field_2", false, -1, leafContext); + // long field 2 descending: + assertLongField("long_field_2", true, numDocs + 1, leafContext); + + // int field 1 ascending: + assertIntField("int_field_1", false, -1, leafContext); + // int field 1 descending: + assertIntField("int_field_1", true, numDocs + 1, leafContext); + // int field 2 ascending: + assertIntField("int_field_2", false, -1, leafContext); + // int field 2 descending: + assertIntField("int_field_2", true, numDocs + 1, leafContext); + + // float field 1 ascending: + assertFloatField("float_field_1", false, -1, leafContext); + // float field 1 descending: + assertFloatField("float_field_1", true, numDocs + 1, leafContext); + // float field 2 ascending: + assertFloatField("float_field_2", false, -1, leafContext); + // float field 2 descending: + assertFloatField("float_field_2", true, numDocs + 1, leafContext); + + // double field 1 ascending: + assertDoubleField("double_field_1", false, -1, leafContext); + // double field 1 descending: + assertDoubleField("double_field_1", true, numDocs + 1, leafContext); + // double field 2 ascending: + assertDoubleField("double_field_2", false, -1, leafContext); + // double field 2 descending: + assertDoubleField("double_field_2", true, numDocs + 1, leafContext); } } } From 2df7908e1523d3ddea581c2bc42e4db5ad5e6992 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 4 Nov 2025 11:56:41 +0100 Subject: [PATCH 5/7] handle missing value --- .../search/comparators/NumericComparator.java | 10 ++++ .../comparators/TestNumericComparator.java | 58 +++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java index 358e23d75205..6abec74319e0 100644 --- a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java +++ b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java @@ -374,6 +374,11 @@ int docCount() { */ void postInitializeCompetitiveIterator() throws IOException { if (queueFull) { + // if some documents have missing points, then check that missing values prohibits + // optimization + if (docCount() < maxDoc && isMissingValueCompetitive()) { + return; + } long bottom = leafComparator.bottomAsComparableLong(); long minValue = sortableBytesToLong(pointValues.getMinPackedValue()); long maxValue = sortableBytesToLong(pointValues.getMaxPackedValue()); @@ -532,6 +537,11 @@ int docCount() { */ void postInitializeCompetitiveIterator() { if (queueFull) { + // if some documents have missing doc values, check that missing values prohibits + // optimization + if (docCount() < maxDoc && isMissingValueCompetitive()) { + return; + } long bottom = leafComparator.bottomAsComparableLong(); if (reverse == false && bottom < skipper.minValue()) { competitiveIterator.update(DocIdSetIterator.empty()); diff --git a/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java b/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java index 8846a9df5f9d..e12878e90f33 100644 --- a/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java +++ b/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java @@ -28,7 +28,10 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Pruning; +import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.tests.util.LuceneTestCase; @@ -105,6 +108,61 @@ public void testEmptyCompetitiveIteratorOptimization() throws Exception { } } + public void testEmptyCompetitiveIteratorOptimizationWithMissingValue() throws Exception { + final int numDocs = atLeast(1000); + try (var dir = newDirectory()) { + try (var writer = + new IndexWriter(dir, new IndexWriterConfig().setMergePolicy(newLogMergePolicy()))) { + // index docs without missing values: + for (int i = 0; i < numDocs; i++) { + var doc = new Document(); + + doc.add(NumericDocValuesField.indexedField("long_field_1", i)); + doc.add(new LongPoint("long_field_2", i)); + doc.add(new NumericDocValuesField("long_field_2", i)); + + writer.addDocument(doc); + if (i % 100 == 0) { + writer.flush(); + } + } + + // Index one doc with without long_field_1 and long_field_2 fields + var doc = new Document(); + doc.add(new NumericDocValuesField("another_field", numDocs)); + writer.addDocument(doc); + writer.flush(); + + try (var reader = DirectoryReader.open(writer)) { + var indexSearcher = newSearcher(reader); + indexSearcher.setQueryCache(null); + { + var sortField = new SortField("long_field_1", SortField.Type.LONG, false); + sortField.setMissingValue(Long.MIN_VALUE); + var topDocs = indexSearcher.search(new MatchAllDocsQuery(), 3, new Sort(sortField)); + assertEquals(numDocs, topDocs.scoreDocs[0].doc); + assertEquals(Long.MIN_VALUE, ((FieldDoc) topDocs.scoreDocs[0]).fields[0]); + assertEquals(0, topDocs.scoreDocs[1].doc); + assertEquals(0L, ((FieldDoc) topDocs.scoreDocs[1]).fields[0]); + assertEquals(1, topDocs.scoreDocs[2].doc); + assertEquals(1L, ((FieldDoc) topDocs.scoreDocs[2]).fields[0]); + } + { + var sortField = new SortField("long_field_2", SortField.Type.LONG, false); + sortField.setMissingValue(Long.MIN_VALUE); + var topDocs = indexSearcher.search(new MatchAllDocsQuery(), 3, new Sort(sortField)); + assertEquals(numDocs, topDocs.scoreDocs[0].doc); + assertEquals(Long.MIN_VALUE, ((FieldDoc) topDocs.scoreDocs[0]).fields[0]); + assertEquals(0, topDocs.scoreDocs[1].doc); + assertEquals(0L, ((FieldDoc) topDocs.scoreDocs[1]).fields[0]); + assertEquals(1, topDocs.scoreDocs[2].doc); + assertEquals(1L, ((FieldDoc) topDocs.scoreDocs[2]).fields[0]); + } + } + } + } + } + private static void assertLongField( String fieldName, boolean reverse, int bottom, LeafReaderContext leafContext) throws IOException { From 63fe7554af9048c3da63603dab4b0d1f8eb18ee3 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 5 Nov 2025 11:01:40 +0100 Subject: [PATCH 6/7] handle hitsThresholdReached --- .../search/comparators/NumericComparator.java | 4 +- .../comparators/TestNumericComparator.java | 44 +++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java index 6abec74319e0..e39ecbbd6661 100644 --- a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java +++ b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java @@ -373,7 +373,7 @@ int docCount() { * @throws IOException i/o exception while fetching min and max values from point values */ void postInitializeCompetitiveIterator() throws IOException { - if (queueFull) { + if (queueFull && hitsThresholdReached) { // if some documents have missing points, then check that missing values prohibits // optimization if (docCount() < maxDoc && isMissingValueCompetitive()) { @@ -536,7 +536,7 @@ int docCount() { * iterator as competitive iterator. */ void postInitializeCompetitiveIterator() { - if (queueFull) { + if (queueFull && hitsThresholdReached) { // if some documents have missing doc values, check that missing values prohibits // optimization if (docCount() < maxDoc && isMissingValueCompetitive()) { diff --git a/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java b/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java index e12878e90f33..88e7c8582272 100644 --- a/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java +++ b/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java @@ -33,7 +33,9 @@ import org.apache.lucene.search.Pruning; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TotalHits; import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.TestUtil; public class TestNumericComparator extends LuceneTestCase { @@ -163,6 +165,44 @@ public void testEmptyCompetitiveIteratorOptimizationWithMissingValue() throws Ex } } + public void testEmptyCompetitiveIteratorOptimizationAndHitsThresholdReached() throws Exception { + final int numDocs = TestUtil.nextInt(random(), 128, 512); // Below IndexSearcher.DEFAULT_HITS_THRESHOLD + try (var dir = newDirectory()) { + try (var writer = + new IndexWriter(dir, new IndexWriterConfig().setMergePolicy(newLogMergePolicy()))) { + for (int i = 0; i < numDocs; i++) { + var doc = new Document(); + + doc.add(NumericDocValuesField.indexedField("field_1", i)); + doc.add(new LongPoint("field_2", i)); + doc.add(new NumericDocValuesField("field_2", i)); + + writer.addDocument(doc); + if (i % 100 == 0) { + writer.flush(); + } + } + + try (var reader = DirectoryReader.open(writer)) { + var indexSearcher = newSearcher(reader); + indexSearcher.setQueryCache(null); + for (String field : new String[] {"field_1", "field_2"}) { + var sortField = new SortField(field, SortField.Type.LONG, false); + var topDocs = indexSearcher.search(new MatchAllDocsQuery(), 3, new Sort(sortField)); + assertEquals(TotalHits.Relation.EQUAL_TO, topDocs.totalHits.relation()); + assertEquals(numDocs, topDocs.totalHits.value()); + assertEquals(0, topDocs.scoreDocs[0].doc); + assertEquals(0L, ((FieldDoc) topDocs.scoreDocs[0]).fields[0]); + assertEquals(1, topDocs.scoreDocs[1].doc); + assertEquals(1L, ((FieldDoc) topDocs.scoreDocs[1]).fields[0]); + assertEquals(2, topDocs.scoreDocs[2].doc); + assertEquals(2L, ((FieldDoc) topDocs.scoreDocs[2]).fields[0]); + } + } + } + } + } + private static void assertLongField( String fieldName, boolean reverse, int bottom, LeafReaderContext leafContext) throws IOException { @@ -171,6 +211,7 @@ private static void assertLongField( new SortField(fieldName, SortField.Type.LONG, reverse) .getComparator(1, Pruning.GREATER_THAN_OR_EQUAL_TO); comparator1.queueFull = true; + comparator1.hitsThresholdReached = true; comparator1.bottom = bottom; var leafComparator = comparator1.getLeafComparator(leafContext); assertEquals(DocIdSetIterator.NO_MORE_DOCS, leafComparator.competitiveIterator().nextDoc()); @@ -184,6 +225,7 @@ private static void assertIntField( new SortField(fieldName, SortField.Type.INT, reverse) .getComparator(1, Pruning.GREATER_THAN_OR_EQUAL_TO); comparator1.queueFull = true; + comparator1.hitsThresholdReached = true; comparator1.bottom = bottom; var leafComparator = comparator1.getLeafComparator(leafContext); assertEquals(DocIdSetIterator.NO_MORE_DOCS, leafComparator.competitiveIterator().nextDoc()); @@ -197,6 +239,7 @@ private static void assertFloatField( new SortField(fieldName, SortField.Type.FLOAT, reverse) .getComparator(1, Pruning.GREATER_THAN_OR_EQUAL_TO); comparator1.queueFull = true; + comparator1.hitsThresholdReached = true; comparator1.bottom = bottom; var leafComparator = comparator1.getLeafComparator(leafContext); assertEquals(DocIdSetIterator.NO_MORE_DOCS, leafComparator.competitiveIterator().nextDoc()); @@ -210,6 +253,7 @@ private static void assertDoubleField( new SortField(fieldName, SortField.Type.DOUBLE, reverse) .getComparator(1, Pruning.GREATER_THAN_OR_EQUAL_TO); comparator1.queueFull = true; + comparator1.hitsThresholdReached = true; comparator1.bottom = bottom; var leafComparator = comparator1.getLeafComparator(leafContext); assertEquals(DocIdSetIterator.NO_MORE_DOCS, leafComparator.competitiveIterator().nextDoc()); From ea4ca25fab5f9c2f5ece2e7cfd4fdd5c409d679b Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 5 Nov 2025 11:11:52 +0100 Subject: [PATCH 7/7] tidy --- .../lucene/search/comparators/TestNumericComparator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java b/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java index 88e7c8582272..417a76c62311 100644 --- a/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java +++ b/lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java @@ -166,7 +166,8 @@ public void testEmptyCompetitiveIteratorOptimizationWithMissingValue() throws Ex } public void testEmptyCompetitiveIteratorOptimizationAndHitsThresholdReached() throws Exception { - final int numDocs = TestUtil.nextInt(random(), 128, 512); // Below IndexSearcher.DEFAULT_HITS_THRESHOLD + final int numDocs = + TestUtil.nextInt(random(), 128, 512); // Below IndexSearcher.DEFAULT_HITS_THRESHOLD try (var dir = newDirectory()) { try (var writer = new IndexWriter(dir, new IndexWriterConfig().setMergePolicy(newLogMergePolicy()))) {