SOLR-13101: merge in 8.3 changes

dev-tools/maven/README.maven

@@ -16,7 +16,7 @@ A. How to use nightly Jenkins-built Lucene/Solr Maven artifacts
    The most recently produced nightly Jenkins-built Lucene and Solr Maven
    snapshot artifacts are available in the Apache Snapshot repository here:
 
-      http://repository.apache.org/snapshots
+      https://repository.apache.org/snapshots
 
    An example POM snippet:
 
@@ -27,7 +27,7 @@ A. How to use nightly Jenkins-built Lucene/Solr Maven artifacts
          <repository>
            <id>apache.snapshots</id>
            <name>Apache Snapshot Repository</name>
-           <url>http://repository.apache.org/snapshots</url>
+           <url>https://repository.apache.org/snapshots</url>
            <releases>
              <enabled>false</enabled>
            </releases>
@@ -57,12 +57,12 @@ C. How to deploy Maven artifacts to a repository
    as in B. above, with the addition of two system properties:
 
       ant -Dm2.repository.id=my-repo-id \
-          -Dm2.repository.url=http://example.org/my/repo \
+          -Dm2.repository.url=https://example.org/my/repo \
           generate-maven-artifacts
 
    The repository ID given in the above command corresponds to a <server>
    entry in either your ~/.m2/settings.xml or ~/.ant/settings.xml.  See
-   <http://maven.apache.org/settings.html#Servers> for more information.
+   <https://maven.apache.org/settings.html#Servers> for more information.
    (Note that as of version 2.1.3, Maven Ant Tasks cannot handle encrypted
    passwords.)
 

dev-tools/maven/pom.xml.template

@@ -32,15 +32,15 @@
   <packaging>pom</packaging>
   <name>Grandparent POM for Apache Lucene Core and Apache Solr</name>
   <description>Grandparent POM for Apache Lucene Core and Apache Solr</description>
-  <url>http://lucene.apache.org</url>
+  <url>https://lucene.apache.org</url>
   <modules>
     <module>lucene</module>
     <module>solr</module>
   </modules>
   <properties>
-    <vc-anonymous-base-url>http://git-wip-us.apache.org/repos/asf/lucene-solr.git</vc-anonymous-base-url>
-    <vc-dev-base-url>https://git-wip-us.apache.org/repos/asf/lucene-solr.git</vc-dev-base-url>
-    <vc-browse-base-url>https://git1-us-west.apache.org/repos/asf?p=lucene-solr.git;a=tree</vc-browse-base-url>
+    <vc-anonymous-base-url>https://gitbox.apache.org/repos/asf/lucene-solr.git</vc-anonymous-base-url>
+    <vc-dev-base-url>https://gitbox.apache.org/repos/asf/lucene-solr.git</vc-dev-base-url>
+    <vc-browse-base-url>https://gitbox.apache.org/repos/asf?p=lucene-solr.git</vc-browse-base-url>
     <specification.version>@spec.version@</specification.version>
     <maven.build.timestamp.format>yyyy-MM-dd HH:mm:ss</maven.build.timestamp.format>
     <java.compat.version>8</java.compat.version>
@@ -82,29 +82,29 @@
       <subscribe>general-subscribe@lucene.apache.org</subscribe>
       <unsubscribe>general-unsubscribe@lucene.apache.org</unsubscribe>
       <archive>
-        http://mail-archives.apache.org/mod_mbox/lucene-general/
+        https://mail-archives.apache.org/mod_mbox/lucene-general/
       </archive>
     </mailingList>
     <mailingList>
       <name>Java User List</name>
       <subscribe>java-user-subscribe@lucene.apache.org</subscribe>
       <unsubscribe>java-user-unsubscribe@lucene.apache.org</unsubscribe>
       <archive>
-        http://mail-archives.apache.org/mod_mbox/lucene-java-user/
+        https://mail-archives.apache.org/mod_mbox/lucene-java-user/
       </archive>
     </mailingList>
     <mailingList>
       <name>Java Developer List</name>
       <subscribe>dev-subscribe@lucene.apache.org</subscribe>
       <unsubscribe>dev-unsubscribe@lucene.apache.org</unsubscribe>
-      <archive>http://mail-archives.apache.org/mod_mbox/lucene-dev/</archive>
+      <archive>https://mail-archives.apache.org/mod_mbox/lucene-dev/</archive>
     </mailingList>
     <mailingList>
       <name>Java Commits List</name>
       <subscribe>commits-subscribe@lucene.apache.org</subscribe>
       <unsubscribe>commits-unsubscribe@lucene.apache.org</unsubscribe>
       <archive>
-        http://mail-archives.apache.org/mod_mbox/lucene-java-commits/
+        https://mail-archives.apache.org/mod_mbox/lucene-java-commits/
       </archive>
     </mailingList>
   </mailingLists>
@@ -124,7 +124,7 @@
     <repository>
       <id>apache.snapshots</id>
       <name>Apache Snapshot Repository</name>
-      <url>http://repository.apache.org/snapshots</url>
+      <url>https://repository.apache.org/snapshots</url>
       <releases>
         <enabled>false</enabled>
       </releases>
@@ -264,7 +264,7 @@
               <tempDir>.</tempDir>
               <java.awt.headless>true</java.awt.headless>
 
-              <!-- See <http://wiki.apache.org/lucene-java/RunningTests>
+              <!-- See <https://cwiki.apache.org/confluence/display/lucene/RunningTests>
                    for a description of the tests.* system properties. -->
 
               <!-- RandomizedTesting library system properties -->

dev-tools/maven/solr/pom.xml.template

@@ -55,21 +55,21 @@
       <subscribe>solr-user-subscribe@lucene.apache.org</subscribe>
       <unsubscribe>solr-user-unsubscribe@lucene.apache.org</unsubscribe>
       <archive>
-        http://mail-archives.apache.org/mod_mbox/solr-user/
+        https://mail-archives.apache.org/mod_mbox/solr-user/
       </archive>
     </mailingList>
     <mailingList>
       <name>Java Developer List</name>
       <subscribe>dev-subscribe@lucene.apache.org</subscribe>
       <unsubscribe>dev-unsubscribe@lucene.apache.org</unsubscribe>
-      <archive>http://mail-archives.apache.org/mod_mbox/lucene-dev/</archive>
+      <archive>https://mail-archives.apache.org/mod_mbox/lucene-dev/</archive>
     </mailingList>
     <mailingList>
       <name>Java Commits List</name>
       <subscribe>commits-subscribe@lucene.apache.org</subscribe>
       <unsubscribe>commits-unsubscribe@lucene.apache.org</unsubscribe>
       <archive>
-        http://mail-archives.apache.org/mod_mbox/lucene-java-commits/
+        https://mail-archives.apache.org/mod_mbox/lucene-java-commits/
       </archive>
     </mailingList>
   </mailingLists>
@@ -78,12 +78,12 @@
     <repository>
       <id>maven-restlet</id>
       <name>Public online Restlet repository</name>
-      <url>http://maven.restlet.org</url>
+      <url>https://maven.restlet.com</url>
     </repository>
     <repository>
       <id>releases.cloudera.com</id>
       <name>Cloudera Releases</name>
-      <url>https://repository.cloudera.com/artifactory/libs-release</url>
+      <url>https://repository.cloudera.com/artifactory/libs-release-local/</url>
     </repository>
   </repositories>
   <build>

dev-tools/scripts/jenkins.build.ref.guide.sh

@@ -61,10 +61,11 @@ echoRun "rvm $RUBY_VERSION@$GEMSET"      # Activate this project's gemset
 # Install gems in the gemset.  Param --force disables dependency conflict detection.
 echoRun "gem install --force --version 3.5.0 jekyll"
 echoRun "gem uninstall --all --ignore-dependencies asciidoctor"  # Get rid of all versions
-echoRun "gem install --force --version 1.5.6.2 asciidoctor"
-echoRun "gem install --force --version 2.1.0 jekyll-asciidoc"
-echoRun "gem install --force --version 1.1.2 pygments.rb"
-echoRun "gem install --force --version 3.0.9 slim"
+echoRun "gem install --force --version 2.0.10 asciidoctor"
+echoRun "gem install --force --version 3.0.0 jekyll-asciidoc"
+echoRun "gem install --force --version 4.0.1 slim"
+echoRun "gem install --force --version 2.0.10 tilt"
+echoRun "gem install --force --version 1.1.5 concurrent-ruby"
 
 cd solr/solr-ref-guide
 

lucene/CHANGES.txt

@@ -18,11 +18,19 @@ API Changes
 * LUCENE-8956: QueryRescorer now only sorts the first topN hits instead of all
   initial hits. (Paul Sanwald via Adrien Grand)
 
+* LUCENE-8921: IndexSearcher.termStatistics() no longer takes a TermStates; it takes the docFreq and totalTermFreq.
+  And don't call if docFreq <= 0.  The previous implementation survives as deprecated and final.  It's removed in 9.0.
+  (Bruno Roustant, David Smiley, Alan Woodward)
+
+* LUCENE-8990: PointValues#estimateDocCount(visitor) estimates the number of documents that would be matched by
+  the given IntersectVisitor. THe method is used to compute the cost() of ScorerSuppliers instead of
+  PointValues#estimatePointCount(visitor). (Ignacio Vera, Adrien Grand)
+
 New Features
 
 * LUCENE-8936: Add SpanishMinimalStemFilter (vinod kumar via Tomoko Uchida)
 
-* LUCENE-8764: Add "export all terms" feature to Luke. (Leonardo Menezes via Tomoko Uchida)
+* LUCENE-8764 LUCENE-8945: Add "export all terms and doc freqs" feature to Luke with delimiters. (Leonardo Menezes, Amish Shah via Tomoko Uchida)
 
 * LUCENE-8747: Composite Matches from multiple subqueries now allow access to
   their submatches, and a new NamedMatches API allows marking of subqueries
@@ -67,6 +75,18 @@ Improvements
 
 * LUCENE-8952: Use a sort key instead of true distance in NearestNeighbor (Julie Tibshirani).
 
+* LUCENE-8620: Tessellator labels the edges of the generated triangles whether they belong to
+  the original polygon. This information is added to the triangle encoding. (Ignacio Vera)
+
+* LUCENE-8964: Fix geojson shape parsing on string arrays in properties
+   (Alexander Reelsen)
+
+* LUCENE-8976: Use exact distance between point and bounding rectangle in FloatPointNearestNeighbor. (Ignacio Vera)
+
+* LUCENE-8966: The Korean analyzer now splits tokens on boundaries between digits and alphabetic characters. (Jim Ferenczi)
+
+* LUCENE-8984: MoreLikeThis MLT is biased for uncommon fields (Andy Hind via Anshum Gupta)
+
 Optimizations
 
 * LUCENE-8922: DisjunctionMaxQuery more efficiently leverages impacts to skip
@@ -81,18 +101,48 @@ the total hits is not requested.
 * LUCENE-8755: spatial-extras quad and packed quad prefix trees now index points faster.
   (Chongchen Chen, David Smiley)
 
+* LUCENE-8860: add additional leaf node level optimizations in LatLonShapeBoundingBoxQuery.
+  (Igor Motov via Ignacio Vera)
+
+* LUCENE-8968: Improve performance of WITHIN and DISJOINT queries for Shape queries by
+  doing just one pass whenever possible. (Ignacio Vera)
+
+* LUCENE-8939: Introduce shared count based early termination across multiple slices
+  (Atri Sharma)
+
+* LUCENE-8980: Blocktree's seekExact now short-circuits false if the term isn't in the min-max range of the segment.
+  Large perf gain for ID/time like data when populated sequentially.  (Guoqiang Jiang)
+
 Bug Fixes
 
 * LUCENE-8755: spatial-extras quad and packed quad prefix trees could throw a
   NullPointerException for certain cell edge coordinates (Chongchen Chen, David Smiley)
 
+* LUCENE-9005: BooleanQuery.visit() would pull subVisitors from its parent visitor, rather
+  than from a visitor for its own specific query.  This could cause problems when BQ was
+  nested under another BQ. Instead, we now pull a MUST subvisitor, pass it to any MUST
+  subclauses, and then pull SHOULD, MUST_NOT and FILTER visitors from it rather than from
+  the parent.  (Alan Woodward)
+
 Other
 
 * LUCENE-8778 LUCENE-8911 LUCENE-8957: Define analyzer SPI names as static final fields and document the names in Javadocs.
   (Tomoko Uchida, Uwe Schindler)
 
 * LUCENE-8758: QuadPrefixTree: removed levelS and levelN fields which weren't used. (Amish Shah)
 
+* LUCENE-8975: Code Cleanup: Use entryset for map iteration wherever possible.
+
+* LUCENE-8993, LUCENE-8807: Changed all repository and download references in build files
+  to HTTPS. (Uwe Schindler)
+
+* LUCENE-8998: Fix OverviewImplTest.testIsOptimized reproducible failure. (Tomoko Uchida)
+
+* LUCENE-8999: LuceneTestCase.expectThrows now propogates assert/assumption failures up to the test
+  w/o wrapping in a new assertion failure unless the caller has explicitly expected them (hossman)
+
+* LUCENE-8062: GlobalOrdinalsWithScoreQuery is no longer eligible for query caching. (Jim Ferenczi)
+
 ======================= Lucene 8.2.0 =======================
 
 API Changes

lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java

@@ -202,7 +202,7 @@ public JapaneseTokenizer(UserDictionary userDictionary, boolean discardPunctuati
   }
 
   /**
-   * Create a new JapaneseTokenizer.
+   * Create a new JapaneseTokenizer using the system and unknown dictionaries shipped with Lucene.
    *
    * @param factory the AttributeFactory to use
    * @param userDictionary Optional: if non-null, user dictionary.
@@ -211,13 +211,41 @@ public JapaneseTokenizer(UserDictionary userDictionary, boolean discardPunctuati
    */
   public JapaneseTokenizer
       (AttributeFactory factory, UserDictionary userDictionary, boolean discardPunctuation, Mode mode) {
+    this(factory,
+         TokenInfoDictionary.getInstance(),
+         UnknownDictionary.getInstance(),
+         ConnectionCosts.getInstance(),
+         userDictionary, discardPunctuation, mode);
+  }
+
+  /**
+   * <p>Create a new JapaneseTokenizer, supplying a custom system dictionary and unknown dictionary.
+   * This constructor provides an entry point for users that want to construct custom language models
+   * that can be used as input to {@link org.apache.lucene.analysis.ja.util.DictionaryBuilder}.</p>
+   *
+   * @param factory the AttributeFactory to use
+   * @param systemDictionary a custom known token dictionary
+   * @param unkDictionary a custom unknown token dictionary
+   * @param connectionCosts custom token transition costs
+   * @param userDictionary Optional: if non-null, user dictionary.
+   * @param discardPunctuation true if punctuation tokens should be dropped from the output.
+   * @param mode tokenization mode.
+   * @lucene.experimental
+   */
+  public JapaneseTokenizer(AttributeFactory factory,
+                           TokenInfoDictionary systemDictionary,
+                           UnknownDictionary unkDictionary,
+                           ConnectionCosts connectionCosts,
+                           UserDictionary userDictionary,
+                           boolean discardPunctuation,
+                           Mode mode) {
     super(factory);
-    dictionary = TokenInfoDictionary.getInstance();
-    fst = dictionary.getFST();
-    unkDictionary = UnknownDictionary.getInstance();
-    characterDefinition = unkDictionary.getCharacterDefinition();
+    this.dictionary = systemDictionary;
+    this.fst = dictionary.getFST();
+    this.unkDictionary = unkDictionary;
+    this.characterDefinition = unkDictionary.getCharacterDefinition();
     this.userDictionary = userDictionary;
-    costs = ConnectionCosts.getInstance();
+    this.costs = connectionCosts;
     fstReader = fst.getBytesReader();
     if (userDictionary != null) {
       userFST = userDictionary.getFST();

lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/BinaryDictionary.java

@@ -176,6 +176,17 @@ protected final InputStream getResource(String suffix) throws IOException {
     }
   }
 
+  public static final InputStream getResource(ResourceScheme scheme, String path) throws IOException {
+    switch(scheme) {
+      case CLASSPATH:
+        return getClassResource(path);
+      case FILE:
+        return Files.newInputStream(Paths.get(path));
+      default:
+        throw new IllegalStateException("unknown resource scheme " + scheme);
+    }
+  }
+
   // util, reused by ConnectionCosts and CharacterDefinition
   public static final InputStream getClassResource(Class<?> clazz, String suffix) throws IOException {
     final InputStream is = clazz.getResourceAsStream(clazz.getSimpleName() + suffix);
@@ -185,7 +196,7 @@ public static final InputStream getClassResource(Class<?> clazz, String suffix)
     return is;
   }
 
-  private InputStream getClassResource(String path) throws IOException {
+  private static InputStream getClassResource(String path) throws IOException {
     final InputStream is = BinaryDictionary.class.getClassLoader().getResourceAsStream(path);
     if (is == null) {
       throw new FileNotFoundException("Not in classpath: " + path);

lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java

@@ -37,12 +37,16 @@ public final class ConnectionCosts {
 
   private final short[][] costs; // array is backward IDs first since get is called using the same backward ID consecutively. maybe doesn't matter.
 
-  private ConnectionCosts() throws IOException {
+  /**
+   * @param scheme - scheme for loading resources (FILE or CLASSPATH).
+   * @param path - where to load resources from, without the ".dat" suffix
+   */
+  public ConnectionCosts(BinaryDictionary.ResourceScheme scheme, String path) throws IOException {
     InputStream is = null;
     short[][] costs = null;
     boolean success = false;
     try {
-      is = BinaryDictionary.getClassResource(getClass(), FILENAME_SUFFIX);
+      is = BinaryDictionary.getResource(scheme, path.replace('.', '/') + FILENAME_SUFFIX);
       is = new BufferedInputStream(is);
       final DataInput in = new InputStreamDataInput(is);
       CodecUtil.checkHeader(in, HEADER, VERSION, VERSION);
@@ -68,7 +72,11 @@ private ConnectionCosts() throws IOException {
 
     this.costs = costs;
   }
-
+
+  private ConnectionCosts() throws IOException {
+    this(BinaryDictionary.ResourceScheme.CLASSPATH, ConnectionCosts.class.getName());
+  }
+
   public int get(int forwardId, int backwardId) {
     return costs[backwardId][forwardId];
   }

lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java

@@ -40,7 +40,7 @@ public final class TokenInfoDictionary extends BinaryDictionary {
    * @param resourcePath - where to load resources (dictionaries) from. If null, with CLASSPATH scheme only, use
    * this class's name as the path.
    */
-  TokenInfoDictionary(ResourceScheme resourceScheme, String resourcePath) throws IOException {
+  public TokenInfoDictionary(ResourceScheme resourceScheme, String resourcePath) throws IOException {
     super(resourceScheme, resourcePath);
     FST<Long> fst;
     try (InputStream is = new BufferedInputStream(getResource(FST_FILENAME_SUFFIX))) {

lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownDictionary.java

@@ -26,6 +26,15 @@ public final class UnknownDictionary extends BinaryDictionary {
 
   private final CharacterDefinition characterDefinition = CharacterDefinition.getInstance();
 
+  /**
+   * @param scheme scheme for loading resources (FILE or CLASSPATH).
+   * @param path where to load resources from; a path, including the file base name without
+   * extension; this is used to match multiple files with the same base name.
+   */
+  public UnknownDictionary(ResourceScheme scheme, String path) throws IOException {
+    super(scheme, path);
+  }
+
   private UnknownDictionary() throws IOException {
     super();
   }