handle missing value

Author: martijnvg

lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java

@@ -374,6 +374,11 @@ int docCount() {
      */
     void postInitializeCompetitiveIterator() throws IOException {
       if (queueFull) {
+        // if some documents have missing points, then check that missing values prohibits
+        // optimization
+        if (docCount() < maxDoc && isMissingValueCompetitive()) {
+          return;
+        }
         long bottom = leafComparator.bottomAsComparableLong();
         long minValue = sortableBytesToLong(pointValues.getMinPackedValue());
         long maxValue = sortableBytesToLong(pointValues.getMaxPackedValue());
@@ -532,6 +537,11 @@ int docCount() {
      */
     void postInitializeCompetitiveIterator() {
       if (queueFull) {
+        // if some documents have missing doc values, check that missing values prohibits
+        // optimization
+        if (docCount() < maxDoc && isMissingValueCompetitive()) {
+          return;
+        }
         long bottom = leafComparator.bottomAsComparableLong();
         if (reverse == false && bottom < skipper.minValue()) {
           competitiveIterator.update(DocIdSetIterator.empty());

lucene/core/src/test/org/apache/lucene/search/comparators/TestNumericComparator.java

@@ -28,7 +28,10 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Pruning;
+import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.tests.util.LuceneTestCase;
 
@@ -105,6 +108,61 @@ public void testEmptyCompetitiveIteratorOptimization() throws Exception {
     }
   }
 
+  public void testEmptyCompetitiveIteratorOptimizationWithMissingValue() throws Exception {
+    final int numDocs = atLeast(1000);
+    try (var dir = newDirectory()) {
+      try (var writer =
+          new IndexWriter(dir, new IndexWriterConfig().setMergePolicy(newLogMergePolicy()))) {
+        // index docs without missing values:
+        for (int i = 0; i < numDocs; i++) {
+          var doc = new Document();
+
+          doc.add(NumericDocValuesField.indexedField("long_field_1", i));
+          doc.add(new LongPoint("long_field_2", i));
+          doc.add(new NumericDocValuesField("long_field_2", i));
+
+          writer.addDocument(doc);
+          if (i % 100 == 0) {
+            writer.flush();
+          }
+        }
+
+        // Index one doc with without long_field_1 and long_field_2 fields
+        var doc = new Document();
+        doc.add(new NumericDocValuesField("another_field", numDocs));
+        writer.addDocument(doc);
+        writer.flush();
+
+        try (var reader = DirectoryReader.open(writer)) {
+          var indexSearcher = newSearcher(reader);
+          indexSearcher.setQueryCache(null);
+          {
+            var sortField = new SortField("long_field_1", SortField.Type.LONG, false);
+            sortField.setMissingValue(Long.MIN_VALUE);
+            var topDocs = indexSearcher.search(new MatchAllDocsQuery(), 3, new Sort(sortField));
+            assertEquals(numDocs, topDocs.scoreDocs[0].doc);
+            assertEquals(Long.MIN_VALUE, ((FieldDoc) topDocs.scoreDocs[0]).fields[0]);
+            assertEquals(0, topDocs.scoreDocs[1].doc);
+            assertEquals(0L, ((FieldDoc) topDocs.scoreDocs[1]).fields[0]);
+            assertEquals(1, topDocs.scoreDocs[2].doc);
+            assertEquals(1L, ((FieldDoc) topDocs.scoreDocs[2]).fields[0]);
+          }
+          {
+            var sortField = new SortField("long_field_2", SortField.Type.LONG, false);
+            sortField.setMissingValue(Long.MIN_VALUE);
+            var topDocs = indexSearcher.search(new MatchAllDocsQuery(), 3, new Sort(sortField));
+            assertEquals(numDocs, topDocs.scoreDocs[0].doc);
+            assertEquals(Long.MIN_VALUE, ((FieldDoc) topDocs.scoreDocs[0]).fields[0]);
+            assertEquals(0, topDocs.scoreDocs[1].doc);
+            assertEquals(0L, ((FieldDoc) topDocs.scoreDocs[1]).fields[0]);
+            assertEquals(1, topDocs.scoreDocs[2].doc);
+            assertEquals(1L, ((FieldDoc) topDocs.scoreDocs[2]).fields[0]);
+          }
+        }
+      }
+    }
+  }
+
   private static void assertLongField(
       String fieldName, boolean reverse, int bottom, LeafReaderContext leafContext)
       throws IOException {