Update Apache Lucene to 9.8.0-snapshot-95cdd2e (#10031)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
opensearch-project · Sep 19, 2023 · 256ea41 · 256ea41
1 parent 2a5b124
commit 256ea41
Show file tree

Hide file tree

Showing 52 changed files with 43 additions and 41 deletions.
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.8.0-snapshot-4373c3b
+lucene            = 9.8.0-snapshot-95cdd2e
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 20.0.2+9

diff --git a/libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 b/libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/libs/core/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 b/libs/core/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+d2f7fbc5b2c49ca777a169d579f41082a9a57cc7
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-4373c3b.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-95cdd2e.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+57e2b0cca55da8ad856dfd60be42e6daabbc98c3
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+0deb3b85eadf831be17b48acab0785fd9d34fc44
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+9a204267d68ce4ba36bfddc366cd6865cf5e1378
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+71e8e811f873ba2b47c7ecf9d890cbeac5b6be41
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+6e1274273895365bd83391cc4b79f5264479f5de
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+e634c8685edad2bdb5c13748b18c0c1a46bb63a3
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-95cdd2e.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+0afdf2afacbae39414ed06325fbb4bed17c07a7d
diff --git a/...ns/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1 b/...ns/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/...ns/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-95cdd2e.jar.sha1 b/...ns/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+166e2ea297182f7bf7070af02aacea9e6a3a19c8
diff --git a/...ava/org/opensearch/plugin/correlation/core/index/mapper/CorrelationVectorFieldMapper.java b/...ava/org/opensearch/plugin/correlation/core/index/mapper/CorrelationVectorFieldMapper.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.plugin.correlation.core.index.mapper;
 
+import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.KnnFloatVectorField;
 import org.apache.lucene.document.StoredField;
@@ -23,16 +24,14 @@
 import java.util.Locale;
 import java.util.Optional;
 
-import static org.apache.lucene.index.FloatVectorValues.MAX_DIMENSIONS;
-
 /**
  * Field mapper for the correlation vector type
  *
  * @opensearch.internal
  */
 public class CorrelationVectorFieldMapper extends VectorFieldMapper {
 
-    private static final int LUCENE_MAX_DIMENSION = MAX_DIMENSIONS;
+    private static final int LUCENE_MAX_DIMENSION = KnnVectorsFormat.DEFAULT_MAX_DIMENSIONS;
 
     private final FieldType vectorFieldType;
 

diff --git a/...java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilderTests.java b/...java/org/opensearch/plugin/correlation/core/index/query/CorrelationQueryBuilderTests.java
@@ -203,7 +203,7 @@ public void testDoToQueryInvalidFieldType() {
 
     /**
      * test serialization of Correlation Query Builder
-     * @throws Exception
+     * @throws Exception Exception
      */
     public void testSerialization() throws Exception {
         assertSerialization(Optional.empty());

diff --git a/server/licenses/lucene-analysis-common-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-analysis-common-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/server/licenses/lucene-analysis-common-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-analysis-common-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+8c82be3d997d781bb72d6d0eadade064dd2cd6db
diff --git a/server/licenses/lucene-backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-backward-codecs-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/server/licenses/lucene-backward-codecs-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-backward-codecs-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+4c261d17c681c0d91171c67e192abfef59adea2e
diff --git a/server/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/server/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+d2f7fbc5b2c49ca777a169d579f41082a9a57cc7
diff --git a/server/licenses/lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-grouping-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/server/licenses/lucene-grouping-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-grouping-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+8d1cf3d6db43fad6630376ba59451f848f4d387c
diff --git a/server/licenses/lucene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-highlighter-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/server/licenses/lucene-highlighter-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-highlighter-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+83ab97638bb5269f950d75bba5675d3cfb63f2fa
diff --git a/server/licenses/lucene-join-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-join-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/server/licenses/lucene-join-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-join-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+97c26362151908dc892263edda3872abbacb71a8
diff --git a/server/licenses/lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-memory-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/server/licenses/lucene-memory-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-memory-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+8337eddc0dddd0d7dd50c5aa0d17e5e31592f9fa
diff --git a/server/licenses/lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-misc-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/server/licenses/lucene-misc-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-misc-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+a2e3fae930295f0e2b401effe04eafc25692a414
diff --git a/server/licenses/lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-queries-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/server/licenses/lucene-queries-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-queries-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+e88d8a464e6cfa345b946c9c8822ba7ee2a9159f
diff --git a/server/licenses/lucene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-queryparser-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/server/licenses/lucene-queryparser-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-queryparser-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+9905790675c01e8dc24f9a5e6b9b28b879c65a52
diff --git a/server/licenses/lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-sandbox-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/server/licenses/lucene-sandbox-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-sandbox-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+d6c8be427ec8ffc7e8233ffbf0d190d95a56cf14
diff --git a/server/licenses/lucene-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-spatial-extras-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/server/licenses/lucene-spatial-extras-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-spatial-extras-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+11716d61288feaa692593bf699affa8de2b564c4
diff --git a/server/licenses/lucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-spatial3d-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/server/licenses/lucene-spatial3d-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-spatial3d-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+3a888e06c0535403b9e58a8dcddeb5e6513a4930
diff --git a/server/licenses/lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1 b/server/licenses/lucene-suggest-9.8.0-snapshot-4373c3b.jar.sha1
diff --git a/server/licenses/lucene-suggest-9.8.0-snapshot-95cdd2e.jar.sha1 b/server/licenses/lucene-suggest-9.8.0-snapshot-95cdd2e.jar.sha1
@@ -0,0 +1 @@
+52dfc8bf135ed29f5baf0a967c1bb63dedb9a069
diff --git a/...sterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java b/...sterTest/java/org/opensearch/action/admin/cluster/node/tasks/ConcurrentSearchTasksIT.java
@@ -108,8 +108,9 @@ public void testConcurrentSearchTaskTracking() {
             assertEquals(mainTaskInfo.getTaskId(), taskInfo.getParentTaskId());
 
             Map<Long, List<ThreadResourceInfo>> threadStats = getThreadStats(SearchAction.NAME + "[*]", taskInfo.getTaskId());
-            // Concurrent search forks each slice of 5 segments to different thread
-            assertEquals((int) Math.ceil(getSegmentCount(INDEX_NAME) / 5.0), threadStats.size());
+            // Concurrent search forks each slice of 5 segments to different thread (see please
+            // https://github.com/apache/lucene/issues/12498)
+            assertEquals((int) Math.ceil(getSegmentCount(INDEX_NAME) / 5.0) + 1, threadStats.size());
 
             // assert that all task descriptions have non-zero length
             MatcherAssert.assertThat(taskInfo.getDescription().length(), greaterThan(0));

diff --git a/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java b/server/src/main/java/org/opensearch/index/engine/TranslogLeafReader.java
@@ -52,7 +52,7 @@
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.VectorEncoding;
 import org.apache.lucene.index.VectorSimilarityFunction;
-import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.KnnCollector;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.opensearch.common.util.set.Sets;
@@ -271,12 +271,12 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException {
     }
 
     @Override
-    public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
+    public void searchNearestVectors(String field, byte[] target, KnnCollector k, Bits acceptDocs) throws IOException {
         throw new UnsupportedOperationException();
     }
 
     @Override
-    public TopDocs searchNearestVectors(String field, byte[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
+    public void searchNearestVectors(String field, float[] target, KnnCollector k, Bits acceptDocs) throws IOException {
         throw new UnsupportedOperationException();
     }
 }
diff --git a/server/src/main/java/org/opensearch/index/search/comparators/UnsignedLongComparator.java b/server/src/main/java/org/opensearch/index/search/comparators/UnsignedLongComparator.java
@@ -86,14 +86,6 @@ public void copy(int slot, int doc) throws IOException {
             super.copy(slot, doc);
         }
 
-        @Override
-        protected boolean isMissingValueCompetitive() {
-            int result = missingValue.compareTo(bottom);
-            // in reverse (desc) sort missingValue is competitive when it's greater or equal to bottom,
-            // in asc sort missingValue is competitive when it's smaller or equal to bottom
-            return reverse ? (result >= 0) : (result <= 0);
-        }
-
         @Override
         protected void encodeBottom(byte[] packedValue) {
             BigIntegerPoint.encodeDimension(bottom, packedValue, 0);
@@ -103,5 +95,15 @@ protected void encodeBottom(byte[] packedValue) {
         protected void encodeTop(byte[] packedValue) {
             BigIntegerPoint.encodeDimension(topValue, packedValue, 0);
         }
+
+        @Override
+        protected int compareMissingValueWithBottomValue() {
+            return missingValue.compareTo(bottom);
+        }
+
+        @Override
+        protected int compareMissingValueWithTopValue() {
+            return missingValue.compareTo(topValue);
+        }
     }
 }