Update to Apache Lucene 9.11.0 (#14074)

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
opensearch-project · Jun 12, 2024 · 4038a3c · 4038a3c
1 parent 81329e9
commit 4038a3c
Show file tree

Hide file tree

Showing 54 changed files with 47 additions and 28 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Added
 
 ### Dependencies
+- Update to Apache Lucene 9.11.0 ([#14042](https://github.com/opensearch-project/OpenSearch/pull/14042))
 
 ### Changed
 

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 opensearch        = 2.16.0
-lucene            = 9.10.0
+lucene            = 9.11.0
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.3+9

diff --git a/libs/core/licenses/lucene-core-9.10.0.jar.sha1 b/libs/core/licenses/lucene-core-9.10.0.jar.sha1
diff --git a/libs/core/licenses/lucene-core-9.11.0.jar.sha1 b/libs/core/licenses/lucene-core-9.11.0.jar.sha1
@@ -0,0 +1 @@
+2e487755a6814b2a1bc770c26569dcba86873dcf
diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java
@@ -133,7 +133,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_14_0 = new Version(2140099, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_14_1 = new Version(2140199, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_15_0 = new Version(2150099, org.apache.lucene.util.Version.LUCENE_9_10_0);
-    public static final Version V_2_16_0 = new Version(2160099, org.apache.lucene.util.Version.LUCENE_9_10_0);
+    public static final Version V_2_16_0 = new Version(2160099, org.apache.lucene.util.Version.LUCENE_9_11_0);
     public static final Version CURRENT = V_2_16_0;
 
     public static Version fromId(int id) {

diff --git a/modules/lang-expression/licenses/lucene-expressions-9.10.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.10.0.jar.sha1
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.11.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.11.0.jar.sha1
@@ -0,0 +1 @@
+5e21d20edee0712472e7c6f605c9d97aeecf16c0
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.10.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.10.0.jar.sha1
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0.jar.sha1
@@ -0,0 +1 @@
+5c7f2d8eab0fca3fdc3d3e57a7f48a335dc7ac33
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.10.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.10.0.jar.sha1
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0.jar.sha1
@@ -0,0 +1 @@
+efcf65dda1b4e9d7e83926fd5895a47e491cbf29
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.10.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.10.0.jar.sha1
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0.jar.sha1
@@ -0,0 +1 @@
+59599d7b8bed2e6bd27d0dad7935c078b98c39cc
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.10.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.10.0.jar.sha1
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0.jar.sha1
@@ -0,0 +1 @@
+e55f83bb373ac139e313f64e80afe1eb0a75b8c0
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.10.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.10.0.jar.sha1
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0.jar.sha1
@@ -0,0 +1 @@
+1be59d91c45a4de069611fb7f8aa3e8fd26020ec
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.10.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.10.0.jar.sha1
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0.jar.sha1
@@ -0,0 +1 @@
+d5b5922acf3743b5a0c542959dd93fca8be333a7
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.10.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.10.0.jar.sha1
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0.jar.sha1
@@ -0,0 +1 @@
+50fd7b471cbdd6648c4972169f3fc67fae9db7f6
diff --git a/server/licenses/lucene-analysis-common-9.10.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.10.0.jar.sha1
diff --git a/server/licenses/lucene-analysis-common-9.11.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.11.0.jar.sha1
@@ -0,0 +1 @@
+75a0a333cf1e043102743066c929e65fe51cbcda
diff --git a/server/licenses/lucene-backward-codecs-9.10.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.10.0.jar.sha1
diff --git a/server/licenses/lucene-backward-codecs-9.11.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.11.0.jar.sha1
@@ -0,0 +1 @@
+db385446bc3fd70e7c6a744276c0a157bd60ee0a
diff --git a/server/licenses/lucene-core-9.10.0.jar.sha1 b/server/licenses/lucene-core-9.10.0.jar.sha1
diff --git a/server/licenses/lucene-core-9.11.0.jar.sha1 b/server/licenses/lucene-core-9.11.0.jar.sha1
@@ -0,0 +1 @@
+2e487755a6814b2a1bc770c26569dcba86873dcf
diff --git a/server/licenses/lucene-grouping-9.10.0.jar.sha1 b/server/licenses/lucene-grouping-9.10.0.jar.sha1
diff --git a/server/licenses/lucene-grouping-9.11.0.jar.sha1 b/server/licenses/lucene-grouping-9.11.0.jar.sha1
@@ -0,0 +1 @@
+882bdaf209b0acb332aa34836616424bcbecf462
diff --git a/server/licenses/lucene-highlighter-9.10.0.jar.sha1 b/server/licenses/lucene-highlighter-9.10.0.jar.sha1
diff --git a/server/licenses/lucene-highlighter-9.11.0.jar.sha1 b/server/licenses/lucene-highlighter-9.11.0.jar.sha1
@@ -0,0 +1 @@
+44accdc03c5482e602718f7bf91e5940ba4e4870
diff --git a/server/licenses/lucene-join-9.10.0.jar.sha1 b/server/licenses/lucene-join-9.10.0.jar.sha1
diff --git a/server/licenses/lucene-join-9.11.0.jar.sha1 b/server/licenses/lucene-join-9.11.0.jar.sha1
@@ -0,0 +1 @@
+32a30ee03ed4f3e43bf63250270b2d4d53050045
diff --git a/server/licenses/lucene-memory-9.10.0.jar.sha1 b/server/licenses/lucene-memory-9.10.0.jar.sha1
diff --git a/server/licenses/lucene-memory-9.11.0.jar.sha1 b/server/licenses/lucene-memory-9.11.0.jar.sha1
@@ -0,0 +1 @@
+b3e80aa6aa3299118e76a23edc23b58f3ba5a515
diff --git a/server/licenses/lucene-misc-9.10.0.jar.sha1 b/server/licenses/lucene-misc-9.10.0.jar.sha1
diff --git a/server/licenses/lucene-misc-9.11.0.jar.sha1 b/server/licenses/lucene-misc-9.11.0.jar.sha1
@@ -0,0 +1 @@
+54fe308908194e1b0697a1157a45c5998c9e1083
diff --git a/server/licenses/lucene-queries-9.10.0.jar.sha1 b/server/licenses/lucene-queries-9.10.0.jar.sha1
diff --git a/server/licenses/lucene-queries-9.11.0.jar.sha1 b/server/licenses/lucene-queries-9.11.0.jar.sha1
@@ -0,0 +1 @@
+987d1286949ddf514b8405fd453ed47bebdfb12d
diff --git a/server/licenses/lucene-queryparser-9.10.0.jar.sha1 b/server/licenses/lucene-queryparser-9.10.0.jar.sha1
diff --git a/server/licenses/lucene-queryparser-9.11.0.jar.sha1 b/server/licenses/lucene-queryparser-9.11.0.jar.sha1
@@ -0,0 +1 @@
+e97fe1c0d102edb8d6e1c01454992fd2b8d80ae0
diff --git a/server/licenses/lucene-sandbox-9.10.0.jar.sha1 b/server/licenses/lucene-sandbox-9.10.0.jar.sha1
diff --git a/server/licenses/lucene-sandbox-9.11.0.jar.sha1 b/server/licenses/lucene-sandbox-9.11.0.jar.sha1
@@ -0,0 +1 @@
+5e46b790744bd9118ccc053f70235364213312a5
diff --git a/server/licenses/lucene-spatial-extras-9.10.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.10.0.jar.sha1
diff --git a/server/licenses/lucene-spatial-extras-9.11.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.11.0.jar.sha1
@@ -0,0 +1 @@
+079ca5aaf544a3acde84b8b88423ace6dedc23eb
diff --git a/server/licenses/lucene-spatial3d-9.10.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.10.0.jar.sha1
diff --git a/server/licenses/lucene-spatial3d-9.11.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.11.0.jar.sha1
@@ -0,0 +1 @@
+564558818d70fc384db5b36fbc8a0ab27b107609
diff --git a/server/licenses/lucene-suggest-9.10.0.jar.sha1 b/server/licenses/lucene-suggest-9.10.0.jar.sha1
diff --git a/server/licenses/lucene-suggest-9.11.0.jar.sha1 b/server/licenses/lucene-suggest-9.11.0.jar.sha1
@@ -0,0 +1 @@
+aa345db9b6caaf881e7890ea5b8911357d592167
diff --git a/server/src/main/java/org/apache/lucene/search/uhighlight/CustomFieldHighlighter.java b/server/src/main/java/org/apache/lucene/search/uhighlight/CustomFieldHighlighter.java
@@ -51,6 +51,8 @@
 class CustomFieldHighlighter extends FieldHighlighter {
     private static final Passage[] EMPTY_PASSAGE = new Passage[0];
 
+    private static final Comparator<Passage> DEFAULT_PASSAGE_SORT_COMPARATOR = Comparator.comparingInt(Passage::getStartOffset);
+
     private final Locale breakIteratorLocale;
     private final int noMatchSize;
     private String fieldValue;
@@ -66,7 +68,16 @@ class CustomFieldHighlighter extends FieldHighlighter {
         PassageFormatter passageFormatter,
         int noMatchSize
     ) {
-        super(field, fieldOffsetStrategy, breakIterator, passageScorer, maxPassages, maxNoHighlightPassages, passageFormatter);
+        super(
+            field,
+            fieldOffsetStrategy,
+            breakIterator,
+            passageScorer,
+            maxPassages,
+            maxNoHighlightPassages,
+            passageFormatter,
+            DEFAULT_PASSAGE_SORT_COMPARATOR
+        );
         this.breakIteratorLocale = breakIteratorLocale;
         this.noMatchSize = noMatchSize;
     }

diff --git a/server/src/main/java/org/opensearch/plugins/PluginsService.java b/server/src/main/java/org/opensearch/plugins/PluginsService.java
@@ -36,6 +36,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.util.SPIClassIterator;
 import org.opensearch.Build;
@@ -762,6 +763,7 @@ static void reloadLuceneSPI(ClassLoader loader) {
         // Codecs:
         PostingsFormat.reloadPostingsFormats(loader);
         DocValuesFormat.reloadDocValuesFormats(loader);
+        KnnVectorsFormat.reloadKnnVectorsFormat(loader);
         Codec.reloadCodecs(loader);
     }
 

diff --git a/...r/src/main/java/org/opensearch/search/fetch/subphase/highlight/FastVectorHighlighter.java b/...r/src/main/java/org/opensearch/search/fetch/subphase/highlight/FastVectorHighlighter.java
@@ -304,7 +304,7 @@ private static BoundaryScanner getBoundaryScanner(Field field) {
                 return DEFAULT_WORD_BOUNDARY_SCANNER;
             case CHARS:
                 if (fieldOptions.boundaryMaxScan() != SimpleBoundaryScanner.DEFAULT_MAX_SCAN
-                    || fieldOptions.boundaryChars() != SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS) {
+                    || fieldOptions.boundaryChars() != HighlightBuilder.DEFAULT_BOUNDARY_CHARS) {
                     return new SimpleBoundaryScanner(fieldOptions.boundaryMaxScan(), fieldOptions.boundaryChars());
                 }
                 return DEFAULT_SIMPLE_BOUNDARY_SCANNER;

diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilder.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightBuilder.java
@@ -111,6 +111,8 @@ public class HighlightBuilder extends AbstractHighlighterBuilder<HighlightBuilde
     /** the default closing tags when {@code tag_schema = "styled"}  */
     public static final String[] DEFAULT_STYLED_POST_TAGS = { "</em>" };
 
+    static final Character[] DEFAULT_BOUNDARY_CHARS = HighlightBuilder.convertCharArray(SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS);
+
     /**
      * a {@link FieldOptions} with default settings
      */
@@ -124,7 +126,7 @@ public class HighlightBuilder extends AbstractHighlighterBuilder<HighlightBuilde
         .numberOfFragments(DEFAULT_NUMBER_OF_FRAGMENTS)
         .encoder(DEFAULT_ENCODER)
         .boundaryMaxScan(SimpleBoundaryScanner.DEFAULT_MAX_SCAN)
-        .boundaryChars(SimpleBoundaryScanner.DEFAULT_BOUNDARY_CHARS)
+        .boundaryChars(DEFAULT_BOUNDARY_CHARS)
         .boundaryScannerLocale(Locale.ROOT)
         .noMatchSize(DEFAULT_NO_MATCH_SIZE)
         .phraseLimit(DEFAULT_PHRASE_LIMIT)

diff --git a/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java
@@ -222,6 +222,9 @@ public abstract class AnalysisFactoryTestCase extends OpenSearchTestCase {
         .put("persianstem", Void.class)
         // https://github.com/apache/lucene/pull/12169
         .put("word2vecsynonym", Void.class)
+        // https://github.com/apache/lucene/pull/12915
+        .put("japanesehiraganauppercase", Void.class)
+        .put("japanesekatakanauppercase", Void.class)
         .immutableMap();
 
     static final Map<String, Class<?>> KNOWN_CHARFILTERS = new MapBuilder<String, Class<?>>()