apache · cpoerschke · Oct 19, 2022 · Oct 7, 2022 · Oct 7, 2022 · Oct 7, 2022
diff --git a/dev-docs/lucene-upgrade.md b/dev-docs/lucene-upgrade.md
@@ -0,0 +1,93 @@
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+ -->
+
+# Lucene upgrade steps
+
+## Read
+
+https://lucene.apache.org/core/9_4_0/MIGRATE.html
+
+## Start
+
+Create a new branch locally e.g. `git checkout -b lucene940 -t origin/main` for upgrading to Lucene 9.4.0 version.
+
+## Build
+
+### `versions.props` update
+
+```
+- org.apache.lucene:*=9.3.0
++ org.apache.lucene:*=9.4.0
+```
+
+### `versions.lock` update
+
+```
+gradlew --write-locks
+```
+
+### `solr/licenses` update
+
+```
+git rm lucene-*-9.3.0.jar.sha1
+
+# ???manually get new .sha1 files???
+
+git add lucene-*-9.4.0.jar.sha1
+```
+
+## Code
+
+```
+gradlew compileJava
+```
+
+* adjust for signature changes e.g.
+  * `int` to `long` type change
+  * additional or removed constructor arguments
+  * additional abstract base class or interface methods
+  * inner classes becoming outer classes
+* codec changes (if any)
+  * conceptually `s/org.apache.lucene.codecs.lucene9x.Lucene9x/org.apache.lucene.codecs.lucene94.Lucene94`
+
+## Test
+
+```
+gradlew compileTestJava
+```
+
+```
+gradlew precommit
+```
+
+```
+gradlew test
+```
+
+## Documentation
+
+### `solr/solr-ref-guide/antora.yml` update
+
+```
+- lucene-javadocs: 'https://lucene.apache.org/core/9_3_0'
++ lucene-javadocs: 'https://lucene.apache.org/core/9_4_0'
+```
+
+## Finish
+
+Push the local branch to github (fork) and open a pull request.
+
diff --git a/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/MMapDirectoryFactory.java
@@ -37,20 +37,20 @@
  *   <li>unmap -- See {@link MMapDirectory#setUseUnmap(boolean)}
  *   <li>preload -- See {@link MMapDirectory#setPreload(boolean)}
  *   <li>maxChunkSize -- The Max chunk size. See {@link MMapDirectory#MMapDirectory(Path,
- *       LockFactory, int)}
+ *       LockFactory, long)}
  * </ul>
  */
 public class MMapDirectoryFactory extends StandardDirectoryFactory {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
   boolean unmapHack;
   boolean preload;
-  private int maxChunk;
+  private long maxChunk;
 
   @Override
   public void init(NamedList<?> args) {
     super.init(args);
     SolrParams params = args.toSolrParams();
-    maxChunk = params.getInt("maxChunkSize", MMapDirectory.DEFAULT_MAX_CHUNK_SIZE);
+    maxChunk = params.getLong("maxChunkSize", MMapDirectory.DEFAULT_MAX_CHUNK_SIZE);
     if (maxChunk <= 0) {
       throw new IllegalArgumentException("maxChunk must be greater than 0");
     }

diff --git a/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java b/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java
@@ -23,9 +23,9 @@
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene92.Lucene92Codec;
-import org.apache.lucene.codecs.lucene92.Lucene92Codec.Mode;
-import org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene94.Lucene94Codec;
+import org.apache.lucene.codecs.lucene94.Lucene94Codec.Mode;
+import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
 import org.apache.solr.common.util.NamedList;
@@ -92,7 +92,7 @@ public void init(NamedList<?> args) {
       log.debug("Using default compressionMode: {}", compressionMode);
     }
     codec =
-        new Lucene92Codec(compressionMode) {
+        new Lucene94Codec(compressionMode) {
           @Override
           public PostingsFormat getPostingsFormatForField(String field) {
             final SchemaField schemaField = core.getLatestSchema().getFieldOrNull(field);
@@ -128,7 +128,7 @@ public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
                 if (knnAlgorithm.equals(DenseVectorField.HNSW_ALGORITHM)) {
                   int maxConn = vectorType.getHnswMaxConn();
                   int beamWidth = vectorType.getHnswBeamWidth();
-                  return new Lucene92HnswVectorsFormat(maxConn, beamWidth);
+                  return new Lucene94HnswVectorsFormat(maxConn, beamWidth);
                 }
               } else {
                 throw new SolrException(

diff --git a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java
@@ -17,8 +17,8 @@
 package org.apache.solr.schema;
 
 import static java.util.Optional.ofNullable;
-import static org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
-import static org.apache.lucene.codecs.lucene92.Lucene92HnswVectorsFormat.DEFAULT_MAX_CONN;
+import static org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
+import static org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat.DEFAULT_MAX_CONN;
 
 import java.util.ArrayList;
 import java.util.List;

diff --git a/solr/core/src/java/org/apache/solr/schema/SchemaField.java b/solr/core/src/java/org/apache/solr/schema/SchemaField.java
@@ -25,6 +25,7 @@
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.IndexableFieldType;
+import org.apache.lucene.index.VectorEncoding;
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.search.SortField;
 import org.apache.solr.common.SolrException;
@@ -548,6 +549,11 @@ public int vectorDimension() {
     return 0;
   }
 
+  @Override
+  public VectorEncoding vectorEncoding() {
+    return VectorEncoding.BYTE;
+  }
+
   @Override
   public VectorSimilarityFunction vectorSimilarityFunction() {
     return VectorSimilarityFunction.EUCLIDEAN;

diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
@@ -522,6 +522,7 @@ private static class ReaderWrapper extends FilterLeafReader {
                   fieldInfo.getPointIndexDimensionCount(),
                   fieldInfo.getPointNumBytes(),
                   fieldInfo.getVectorDimension(),
+                  fieldInfo.getVectorEncoding(),
                   fieldInfo.getVectorSimilarityFunction(),
                   fieldInfo.isSoftDeletesField());
           newInfos.add(f);

diff --git a/solr/core/src/java/org/apache/solr/search/NumericHidingLeafReader.java b/solr/core/src/java/org/apache/solr/search/NumericHidingLeafReader.java
@@ -74,6 +74,7 @@ private NumericHidingLeafReader(LeafReader in, String field) {
                 fi.getPointIndexDimensionCount(),
                 fi.getPointNumBytes(),
                 fi.getVectorDimension(),
+                fi.getVectorEncoding(),
                 fi.getVectorSimilarityFunction(),
                 fi.isSoftDeletesField()));
       } else {

diff --git a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java
@@ -296,6 +296,7 @@ public static LeafReader wrap(LeafReader in, Function<String, Type> mapping) {
                 fi.getPointIndexDimensionCount(),
                 fi.getPointNumBytes(),
                 fi.getVectorDimension(),
+                fi.getVectorEncoding(),
                 fi.getVectorSimilarityFunction(),
                 fi.isSoftDeletesField()));
       } else {

diff --git a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java
@@ -21,7 +21,7 @@
 import java.io.IOException;
 import java.util.Map;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene92.Lucene92Codec.Mode;
+import org.apache.lucene.codecs.lucene94.Lucene94Codec.Mode;
 import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
 import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
 import org.apache.lucene.index.SegmentInfo;

diff --git a/solr/licenses/lucene-analysis-common-9.3.0.jar.sha1 b/solr/licenses/lucene-analysis-common-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-analysis-common-9.4.0.jar.sha1 b/solr/licenses/lucene-analysis-common-9.4.0.jar.sha1
@@ -0,0 +1 @@
+02fbd4e87241411fcf5d34e92a50bee46ab164dc
diff --git a/solr/licenses/lucene-analysis-icu-9.3.0.jar.sha1 b/solr/licenses/lucene-analysis-icu-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-analysis-icu-9.4.0.jar.sha1 b/solr/licenses/lucene-analysis-icu-9.4.0.jar.sha1
@@ -0,0 +1 @@
+aa0f250558375922f3091820361156e514fe1842
diff --git a/solr/licenses/lucene-analysis-kuromoji-9.3.0.jar.sha1 b/solr/licenses/lucene-analysis-kuromoji-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-analysis-kuromoji-9.4.0.jar.sha1 b/solr/licenses/lucene-analysis-kuromoji-9.4.0.jar.sha1
@@ -0,0 +1 @@
+32eb1ad367ab1289804aeed95ea7216711a7764d
diff --git a/solr/licenses/lucene-analysis-morfologik-9.3.0.jar.sha1 b/solr/licenses/lucene-analysis-morfologik-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-analysis-morfologik-9.4.0.jar.sha1 b/solr/licenses/lucene-analysis-morfologik-9.4.0.jar.sha1
@@ -0,0 +1 @@
+13e1ae2c760d8c0d7990ffe3296e46d9d8e6f842
diff --git a/solr/licenses/lucene-analysis-nori-9.3.0.jar.sha1 b/solr/licenses/lucene-analysis-nori-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-analysis-nori-9.4.0.jar.sha1 b/solr/licenses/lucene-analysis-nori-9.4.0.jar.sha1
@@ -0,0 +1 @@
+63661714be65f882a921d281965b0779fd487b90
diff --git a/solr/licenses/lucene-analysis-opennlp-9.3.0.jar.sha1 b/solr/licenses/lucene-analysis-opennlp-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-analysis-opennlp-9.4.0.jar.sha1 b/solr/licenses/lucene-analysis-opennlp-9.4.0.jar.sha1
@@ -0,0 +1 @@
+c0ca97d8f89e674084b44240d587453cf5df8670
diff --git a/solr/licenses/lucene-analysis-phonetic-9.3.0.jar.sha1 b/solr/licenses/lucene-analysis-phonetic-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-analysis-phonetic-9.4.0.jar.sha1 b/solr/licenses/lucene-analysis-phonetic-9.4.0.jar.sha1
@@ -0,0 +1 @@
+1034d876551fc21f7835b456dab01db21b9a4af6
diff --git a/solr/licenses/lucene-analysis-smartcn-9.3.0.jar.sha1 b/solr/licenses/lucene-analysis-smartcn-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-analysis-smartcn-9.4.0.jar.sha1 b/solr/licenses/lucene-analysis-smartcn-9.4.0.jar.sha1
@@ -0,0 +1 @@
+f704ee4b14e2fe2622bb983f04b36a32df8fd4a7
diff --git a/solr/licenses/lucene-analysis-stempel-9.3.0.jar.sha1 b/solr/licenses/lucene-analysis-stempel-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-analysis-stempel-9.4.0.jar.sha1 b/solr/licenses/lucene-analysis-stempel-9.4.0.jar.sha1
@@ -0,0 +1 @@
+a95ff17b51da6b3da641fa4053e5ee9ea2ff5daf
diff --git a/solr/licenses/lucene-backward-codecs-9.3.0.jar.sha1 b/solr/licenses/lucene-backward-codecs-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-backward-codecs-9.4.0.jar.sha1 b/solr/licenses/lucene-backward-codecs-9.4.0.jar.sha1
@@ -0,0 +1 @@
+259863dfd107645de6146b3c87b4ecee66a4d43d
diff --git a/solr/licenses/lucene-classification-9.3.0.jar.sha1 b/solr/licenses/lucene-classification-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-classification-9.4.0.jar.sha1 b/solr/licenses/lucene-classification-9.4.0.jar.sha1
@@ -0,0 +1 @@
+35fb746cbfc8a36264a2e6352294d8a45c9006fe
diff --git a/solr/licenses/lucene-codecs-9.3.0.jar.sha1 b/solr/licenses/lucene-codecs-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-codecs-9.4.0.jar.sha1 b/solr/licenses/lucene-codecs-9.4.0.jar.sha1
@@ -0,0 +1 @@
+5c9d00dbe95e88980938f20be1e130070dd03b3a
diff --git a/solr/licenses/lucene-core-9.3.0.jar.sha1 b/solr/licenses/lucene-core-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-core-9.4.0.jar.sha1 b/solr/licenses/lucene-core-9.4.0.jar.sha1
@@ -0,0 +1 @@
+cca1116f813c0f0c63acfac4c952baf29d46d76b
diff --git a/solr/licenses/lucene-expressions-9.3.0.jar.sha1 b/solr/licenses/lucene-expressions-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-expressions-9.4.0.jar.sha1 b/solr/licenses/lucene-expressions-9.4.0.jar.sha1
@@ -0,0 +1 @@
+19749e264805171009836cbedecc5494b13cd920
diff --git a/solr/licenses/lucene-grouping-9.3.0.jar.sha1 b/solr/licenses/lucene-grouping-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-grouping-9.4.0.jar.sha1 b/solr/licenses/lucene-grouping-9.4.0.jar.sha1
@@ -0,0 +1 @@
+51bec1d5acc8ecaf9f50e047d3f86d60c7a958f4
diff --git a/solr/licenses/lucene-highlighter-9.3.0.jar.sha1 b/solr/licenses/lucene-highlighter-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-highlighter-9.4.0.jar.sha1 b/solr/licenses/lucene-highlighter-9.4.0.jar.sha1
@@ -0,0 +1 @@
+c8cf8c9308d8fb18a927c7ed267a14ace3990a5f
diff --git a/solr/licenses/lucene-join-9.3.0.jar.sha1 b/solr/licenses/lucene-join-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-join-9.4.0.jar.sha1 b/solr/licenses/lucene-join-9.4.0.jar.sha1
@@ -0,0 +1 @@
+99b2d3c8e137a6853a2503456897d47d4f18974b
diff --git a/solr/licenses/lucene-memory-9.3.0.jar.sha1 b/solr/licenses/lucene-memory-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-memory-9.4.0.jar.sha1 b/solr/licenses/lucene-memory-9.4.0.jar.sha1
@@ -0,0 +1 @@
+881cb214e79da14de35cb0e8e6779d2722828a96
diff --git a/solr/licenses/lucene-misc-9.3.0.jar.sha1 b/solr/licenses/lucene-misc-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-misc-9.4.0.jar.sha1 b/solr/licenses/lucene-misc-9.4.0.jar.sha1
@@ -0,0 +1 @@
+a126123e482e6bf2e7aea670d221a2a39d3277dc
diff --git a/solr/licenses/lucene-queries-9.3.0.jar.sha1 b/solr/licenses/lucene-queries-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-queries-9.4.0.jar.sha1 b/solr/licenses/lucene-queries-9.4.0.jar.sha1
@@ -0,0 +1 @@
+fe74dbfe9dba9ee9ee2cb80f151fde97fb4efd12
diff --git a/solr/licenses/lucene-queryparser-9.3.0.jar.sha1 b/solr/licenses/lucene-queryparser-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-queryparser-9.4.0.jar.sha1 b/solr/licenses/lucene-queryparser-9.4.0.jar.sha1
@@ -0,0 +1 @@
+13f108a8572fcf0670c7df3ba8dbe1076d0e0dbe
diff --git a/solr/licenses/lucene-sandbox-9.3.0.jar.sha1 b/solr/licenses/lucene-sandbox-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-sandbox-9.4.0.jar.sha1 b/solr/licenses/lucene-sandbox-9.4.0.jar.sha1
@@ -0,0 +1 @@
+e7a676a12ea50dcbf64564f4e4022f939f0a627d
diff --git a/solr/licenses/lucene-spatial-extras-9.3.0.jar.sha1 b/solr/licenses/lucene-spatial-extras-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-spatial-extras-9.4.0.jar.sha1 b/solr/licenses/lucene-spatial-extras-9.4.0.jar.sha1
@@ -0,0 +1 @@
+84d956d1cb1458c51967af1c4acadd2a1f92634d
diff --git a/solr/licenses/lucene-spatial3d-9.3.0.jar.sha1 b/solr/licenses/lucene-spatial3d-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-spatial3d-9.4.0.jar.sha1 b/solr/licenses/lucene-spatial3d-9.4.0.jar.sha1
@@ -0,0 +1 @@
+76887ca708f23b13613e45fb9e307c548b22c6da
diff --git a/solr/licenses/lucene-suggest-9.3.0.jar.sha1 b/solr/licenses/lucene-suggest-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-suggest-9.4.0.jar.sha1 b/solr/licenses/lucene-suggest-9.4.0.jar.sha1
@@ -0,0 +1 @@
+406c9c539f262449d3b1e57e7bc4302efeecaf6c
diff --git a/solr/licenses/lucene-test-framework-9.3.0.jar.sha1 b/solr/licenses/lucene-test-framework-9.3.0.jar.sha1
diff --git a/solr/licenses/lucene-test-framework-9.4.0.jar.sha1 b/solr/licenses/lucene-test-framework-9.4.0.jar.sha1
@@ -0,0 +1 @@
+5f13400741a511fd4ba48e64d5637fb0a8b1bd7a
diff --git a/solr/server/solr/configsets/_default/conf/solrconfig.xml b/solr/server/solr/configsets/_default/conf/solrconfig.xml
@@ -35,7 +35,7 @@
        that you fully re-index after changing this setting as it can
        affect both how text is indexed and queried.
   -->
-  <luceneMatchVersion>9.3</luceneMatchVersion>
+  <luceneMatchVersion>9.4</luceneMatchVersion>
 
   <!-- <lib/> directives can be used to instruct Solr to load any Jars
        identified and use them to resolve any "plugins" specified in

diff --git a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
@@ -35,7 +35,7 @@
        that you fully re-index after changing this setting as it can
        affect both how text is indexed and queried.
   -->
-  <luceneMatchVersion>9.3</luceneMatchVersion>
+  <luceneMatchVersion>9.4</luceneMatchVersion>
 
   <!-- <lib/> directives can be used to instruct Solr to load any Jars
        identified and use them to resolve any "plugins" specified in

diff --git a/solr/solr-ref-guide/antora.yml b/solr/solr-ref-guide/antora.yml
@@ -35,7 +35,7 @@ asciidoc:
     # 'patch-version' is the 'z' part of x.y.z semantic version
     page-solr-javadocs: 'https://solr.apache.org/docs/10_0_0'
     solr-javadocs: 'https://solr.apache.org/docs/10_0_0'
-    lucene-javadocs: 'https://lucene.apache.org/core/9_3_0'
+    lucene-javadocs: 'https://lucene.apache.org/core/9_4_0'
     java-javadocs: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/
     # Some constructed  links may be 404 TODO to check
     solr-docs-version: '10.0'

diff --git a/solr/test-framework/src/java/org/apache/solr/schema/SortableBinaryField.java b/solr/test-framework/src/java/org/apache/solr/schema/SortableBinaryField.java
@@ -23,9 +23,9 @@
 import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.search.FieldComparator;
 import org.apache.lucene.search.FieldComparatorSource;
 import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.comparators.TermOrdValComparator;
 import org.apache.lucene.util.BytesRef;
 
 /** Custom field representing a {@link BinaryField} that's sortable. */
@@ -70,12 +70,14 @@ public BinarySortField(final String field, final boolean reverse) {
           field,
           new FieldComparatorSource() {
             @Override
-            public FieldComparator.TermOrdValComparator newComparator(
+            public TermOrdValComparator newComparator(
                 final String fieldname,
                 final int numHits,
                 final boolean enableSkipping,
                 final boolean reversed) {
-              return new FieldComparator.TermOrdValComparator(numHits, fieldname);
+              final boolean sortMissingLast = false;
+              return new TermOrdValComparator(
+                  numHits, fieldname, sortMissingLast, reversed, enableSkipping);
             }
           },
           reverse);