Upgrade to Lucene 9.2 snapshot efa5d6f4d43 (#86227)

Notable changes include:

count implementations for MultiRangeQuery and IndexSortedNumericDocValuesRangeQuery, which may speed up certain aggregations
more efficient decoding of docids in BKD reader

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.3.0
-lucene            = 9.1.0
+lucene            = 9.2.0-snapshot-efa5d6f4d43
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 18+36

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        9.1.0
-:lucene_version_path:   9_1_0
+:lucene_version:        9.2.0
+:lucene_version_path:   9_2_0
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

docs/changelog/86227.yaml

@@ -0,0 +1,5 @@
+pr: 86227
+summary: Upgrade to Lucene 9.2 snapshot
+area: Search
+type: upgrade
+issues: []

libs/core/src/test/java/org/elasticsearch/core/IOUtilsTests.java

@@ -26,7 +26,6 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.net.URI;
 import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.AccessDeniedException;
@@ -158,10 +157,8 @@ public void runTestRm(final boolean exception) throws IOException {
         for (int i = 0; i < numberOfLocations; i++) {
             if (exception && randomBoolean()) {
                 final Path location = createTempDir();
-                final FileSystem fs = new AccessDeniedWhileDeletingFileSystem(location.getFileSystem()).getFileSystem(
-                    URI.create("file:///")
-                );
-                final Path wrapped = new FilterPath(location, fs);
+                final FilterFileSystemProvider fsProvider = new AccessDeniedWhileDeletingFileSystem(location.getFileSystem());
+                final Path wrapped = fsProvider.wrapPath(location);
                 locations[i] = wrapped.resolve(randomAlphaOfLength(8));
                 Files.createDirectory(locations[i]);
                 locationsThrowingException.add(locations[i]);
@@ -246,8 +243,8 @@ public FileChannel newFileChannel(final Path path, final Set<? extends OpenOptio
 
     public void testFsyncAccessDeniedOpeningDirectory() throws Exception {
         final Path path = createTempDir().toRealPath();
-        final FileSystem fs = new AccessDeniedWhileOpeningDirectoryFileSystem(path.getFileSystem()).getFileSystem(URI.create("file:///"));
-        final Path wrapped = new FilterPath(path, fs);
+        final FilterFileSystemProvider fsProvider = new AccessDeniedWhileOpeningDirectoryFileSystem(path.getFileSystem());
+        final Path wrapped = fsProvider.wrapPath(path);
         if (Constants.WINDOWS) {
             // no exception, we early return and do not even try to open the directory
             fsync(wrapped, true);

modules/lang-expression/licenses/lucene-expressions-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+1755a2c955a53ae73096ec2b72b8982d676554b9

modules/legacy-geo/licenses/lucene-spatial-extras-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+563fc02413da6a9e9f6b5afdaf92df66253d16ce

plugins/analysis-icu/licenses/lucene-analysis-icu-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+1943e5a73bfed85067862388e931bb1702764547

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+aa21165b2cbc19ba0cc0b0b3ec675ca2577de80d

plugins/analysis-nori/licenses/lucene-analysis-nori-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+57e5f7a8a10ae291e94db06d0262a757d08952dd

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+350720d6ebb8cb0b6296069a35f68d4081992b46

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+ec86d5ef7d47ae178dd34f52c0df39bd9b9c4344

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+43ffa0c82753b644b66dfc931cae50ca0d351492

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+8fab9f95101a853faaf86eef58f43f5517e5b679

server/licenses/lucene-analysis-common-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+38f13d4a20b4a9aec0307004ee18b87845c5e6e8

server/licenses/lucene-backward-codecs-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+c9ac8533aa40378755539d3722b1ffed08b53b7c

server/licenses/lucene-core-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+8be8ebe2a770686fede2c213fd81d4bb0b9f1590

server/licenses/lucene-grouping-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+686821eb6732242e3bedebd67788ded567b63470

server/licenses/lucene-highlighter-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+5d7032e3c10e79e907748478cb1662335ce42345

server/licenses/lucene-join-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+2a249b6127d5c05c67e16d3e1d3a542d5440629c

server/licenses/lucene-memory-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+ad395a9f5a29b793bd431d790bb34bac9dce3b80

server/licenses/lucene-misc-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+08f39a2036656e70844d68227df79f7eb160d171

server/licenses/lucene-queries-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+beb6b98e38261d349da54d7cca367847925a2646

server/licenses/lucene-queryparser-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+5adb8b965bae11f0aa3ce7fa43558d6d1eecffeb

server/licenses/lucene-sandbox-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+22acf5c5f112bed98fa4dfffbf09d5f17ec298e5

server/licenses/lucene-spatial3d-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+f57269c7c419a5046792da61cfd1d532f9914390

server/licenses/lucene-suggest-9.2.0-snapshot-efa5d6f4d43.jar.sha1

@@ -0,0 +1 @@
+0ac869cf9e6a5045b2578a0f7fbf2acc88f3ceaf

server/src/main/java/org/elasticsearch/Version.java

@@ -109,7 +109,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_8_1_3 = new Version(8_01_03_99, org.apache.lucene.util.Version.LUCENE_9_0_0);
     public static final Version V_8_2_0 = new Version(8_02_00_99, org.apache.lucene.util.Version.LUCENE_9_1_0);
     public static final Version V_8_2_1 = new Version(8_02_01_99, org.apache.lucene.util.Version.LUCENE_9_1_0);
-    public static final Version V_8_3_0 = new Version(8_03_00_99, org.apache.lucene.util.Version.LUCENE_9_1_0);
+    public static final Version V_8_3_0 = new Version(8_03_00_99, org.apache.lucene.util.Version.LUCENE_9_2_0);
     public static final Version CURRENT = V_8_3_0;
 
     private static final Map<Integer, Version> idToVersion;

server/src/main/java/org/elasticsearch/index/engine/LazySoftDeletesDirectoryReaderWrapper.java

@@ -11,6 +11,7 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.CodecReader;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FilterCodecReader;
 import org.apache.lucene.index.FilterDirectoryReader;
 import org.apache.lucene.index.FilterLeafReader;
@@ -23,7 +24,6 @@
 import org.apache.lucene.index.SoftDeletesRetentionMergePolicy;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.common.lucene.Lucene;
@@ -227,7 +227,7 @@ public int length() {
         private Bits init() throws IOException {
             assert Thread.holdsLock(this);
 
-            DocIdSetIterator iterator = DocValuesFieldExistsQuery.getDocValuesDocIdSetIterator(field, reader);
+            DocIdSetIterator iterator = getDocValuesDocIdSetIterator(field, reader);
             assert iterator != null;
             Bits liveDocs = reader.getLiveDocs();
             final FixedBitSet bits;
@@ -274,4 +274,39 @@ private static class DelegatingCacheHelper extends org.apache.lucene.index.Filte
             super(delegate);
         }
     }
+
+    /**
+     * Returns a {@link DocIdSetIterator} from the given field or null if the field doesn't exist in
+     * the reader or if the reader has no doc values for the field.
+     */
+    private static DocIdSetIterator getDocValuesDocIdSetIterator(String field, LeafReader reader) throws IOException {
+        FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
+        final DocIdSetIterator iterator;
+        if (fieldInfo != null) {
+            switch (fieldInfo.getDocValuesType()) {
+                case NONE:
+                    iterator = null;
+                    break;
+                case NUMERIC:
+                    iterator = reader.getNumericDocValues(field);
+                    break;
+                case BINARY:
+                    iterator = reader.getBinaryDocValues(field);
+                    break;
+                case SORTED:
+                    iterator = reader.getSortedDocValues(field);
+                    break;
+                case SORTED_NUMERIC:
+                    iterator = reader.getSortedNumericDocValues(field);
+                    break;
+                case SORTED_SET:
+                    iterator = reader.getSortedSetDocValues(field);
+                    break;
+                default:
+                    throw new AssertionError();
+            }
+            return iterator;
+        }
+        return null;
+    }
 }

server/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalMapping.java

@@ -59,6 +59,11 @@ public long nextOrd() throws IOException {
         }
     }
 
+    @Override
+    public long docValueCount() {
+        return values.docValueCount();
+    }
+
     @Override
     public BytesRef lookupOrd(long globalOrd) throws IOException {
         final long segmentOrd = ordinalMap.getFirstSegmentOrd(globalOrd);

server/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java

@@ -189,6 +189,11 @@ public long nextOrd() throws IOException {
             }
         }
 
+        @Override
+        public long docValueCount() {
+            return currentEndOffset - currentOffset;
+        }
+
         @Override
         public BytesRef lookupOrd(long ord) {
             return values.lookupOrd(ord);

server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java

@@ -459,6 +459,11 @@ public long nextOrd() {
                 return i;
             }
 
+            @Override
+            public long docValueCount() {
+                return values.size();
+            }
+
             @Override
             public BytesRef lookupOrd(long ord) {
                 return values.get((int) ord);

server/src/main/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldData.java

@@ -209,6 +209,11 @@ public long nextOrd() throws IOException {
             }
         }
 
+        @Override
+        public long docValueCount() {
+            return delegate.docValueCount();
+        }
+
         @Override
         public boolean advanceExact(int target) throws IOException {
             if (delegate.advanceExact(target)) {

server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java

@@ -275,6 +275,11 @@ public long nextOrd() throws IOException {
                 }
             }
 
+            @Override
+            public long docValueCount() {
+                return values.docValueCount();
+            }
+
             @Override
             public boolean advanceExact(int doc) throws IOException {
                 hasOrds = values.advanceExact(doc);
@@ -309,6 +314,11 @@ public BytesRef lookupOrd(long ord) throws IOException {
                 }
             }
 
+            @Override
+            public long docValueCount() {
+                return values.docValueCount();
+            }
+
             @Override
             public long getValueCount() {
                 return 1 + values.getValueCount();

server/src/test/java/org/elasticsearch/index/mapper/flattened/KeyedFlattenedLeafFieldDataTests.java

@@ -199,6 +199,11 @@ public long nextOrd() {
             return documentOrds[index++];
         }
 
+        @Override
+        public long docValueCount() {
+            return documentOrds.length;
+        }
+
         @Override
         public BytesRef lookupOrd(long ord) {
             return allTerms[(int) ord];

server/src/test/java/org/elasticsearch/search/MultiValueModeTests.java

@@ -723,6 +723,11 @@ public long nextOrd() {
                 }
             }
 
+            @Override
+            public long docValueCount() {
+                return array[doc].length;
+            }
+
             @Override
             public boolean advanceExact(int docID) {
                 this.doc = docID;

server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java

@@ -876,7 +876,7 @@ public void testOneBucketOptimized() throws IOException {
                                             .entry(
                                                 "filters",
                                                 matchesList().item(
-                                                    matchesMap().entry("query", "DocValuesFieldExistsQuery [field=f]")
+                                                    matchesMap().entry("query", "FieldExistsQuery [field=f]")
                                                         .entry("specialized_for", "docvalues_field_exists")
                                                         .entry("results_from_metadata", greaterThan(0))
                                                 )

server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java

@@ -47,6 +47,11 @@ public long nextOrd() {
             return ords[i++];
         }
 
+        @Override
+        public long docValueCount() {
+            return ords.length;
+        }
+
         @Override
         public BytesRef lookupOrd(long ord) {
             return terms[(int) ord];

server/src/test/java/org/elasticsearch/search/aggregations/support/IncludeExcludeTests.java

@@ -66,6 +66,11 @@ public long nextOrd() {
                 }
             }
 
+            @Override
+            public long docValueCount() {
+                return 1;
+            }
+
             @Override
             public BytesRef lookupOrd(long ord) {
                 assertEquals(0, ord);