LUCENE-10335: IOUtils.getDecodingReader(Class<?>, String) is broken w…

…ith modules/
dweiss · Dec 23, 2021 · 7743921 · 7743921
1 parent 9fe2855
commit 7743921
Show file tree

Hide file tree

Showing 21 changed files with 66 additions and 21 deletions.
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java
@@ -65,7 +65,8 @@ private static class DefaultSetHolder {
         DEFAULT_STOP_SET =
             WordlistLoader.getWordSet(
                 IOUtils.getDecodingReader(
-                    BrazilianAnalyzer.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8),
+                    () -> BrazilianAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                    StandardCharsets.UTF_8),
                 "#");
       } catch (IOException ex) {
         // default set should always be present as it is part of the

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniAnalyzer.java
@@ -65,7 +65,8 @@ private static class DefaultSetHolder {
         DEFAULT_STOP_SET =
             WordlistLoader.getWordSet(
                 IOUtils.getDecodingReader(
-                    SoraniAnalyzer.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
+                    () -> SoraniAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                    StandardCharsets.UTF_8));
       } catch (IOException ex) {
         // default set should always be present as it is part of the
         // distribution (JAR)

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java
@@ -61,7 +61,8 @@ private static class DefaultSetHolder {
         DEFAULT_SET =
             WordlistLoader.getWordSet(
                 IOUtils.getDecodingReader(
-                    CzechAnalyzer.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8),
+                    () -> CzechAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                    StandardCharsets.UTF_8),
                 "#");
       } catch (IOException ex) {
         // default set should always be present as it is part of the

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/da/DanishAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/da/DanishAnalyzer.java
@@ -66,7 +66,8 @@ private static class DefaultSetHolder {
         DEFAULT_STOP_SET =
             WordlistLoader.getSnowballWordSet(
                 IOUtils.getDecodingReader(
-                    SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
+                    () -> SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                    StandardCharsets.UTF_8));
       } catch (IOException ex) {
         // default set should always be present as it is part of the
         // distribution (JAR)

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java
@@ -76,7 +76,8 @@ private static class DefaultSetHolder {
         DEFAULT_SET =
             WordlistLoader.getSnowballWordSet(
                 IOUtils.getDecodingReader(
-                    SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
+                    () -> SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                    StandardCharsets.UTF_8));
       } catch (IOException ex) {
         // default set should always be present as it is part of the
         // distribution (JAR)

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishAnalyzer.java
@@ -65,7 +65,8 @@ private static class DefaultSetHolder {
         DEFAULT_STOP_SET =
             WordlistLoader.getSnowballWordSet(
                 IOUtils.getDecodingReader(
-                    SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
+                    () -> SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                    StandardCharsets.UTF_8));
       } catch (IOException ex) {
         // default set should always be present as it is part of the
         // distribution (JAR)

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishAnalyzer.java
@@ -66,7 +66,8 @@ private static class DefaultSetHolder {
         DEFAULT_STOP_SET =
             WordlistLoader.getSnowballWordSet(
                 IOUtils.getDecodingReader(
-                    SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
+                    () -> SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                    StandardCharsets.UTF_8));
       } catch (IOException ex) {
         // default set should always be present as it is part of the
         // distribution (JAR)

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java
@@ -82,7 +82,8 @@ private static class DefaultSetHolder {
         DEFAULT_STOP_SET =
             WordlistLoader.getSnowballWordSet(
                 IOUtils.getDecodingReader(
-                    SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
+                    () -> SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                    StandardCharsets.UTF_8));
       } catch (IOException ex) {
         // default set should always be present as it is part of the
         // distribution (JAR)

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/gl/GalicianAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/gl/GalicianAnalyzer.java
@@ -63,7 +63,8 @@ private static class DefaultSetHolder {
         DEFAULT_STOP_SET =
             WordlistLoader.getWordSet(
                 IOUtils.getDecodingReader(
-                    GalicianAnalyzer.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
+                    () -> GalicianAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                    StandardCharsets.UTF_8));
       } catch (IOException ex) {
         // default set should always be present as it is part of the
         // distribution (JAR)

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianAnalyzer.java
@@ -66,7 +66,8 @@ private static class DefaultSetHolder {
         DEFAULT_STOP_SET =
             WordlistLoader.getSnowballWordSet(
                 IOUtils.getDecodingReader(
-                    SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
+                    () -> SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                    StandardCharsets.UTF_8));
       } catch (IOException ex) {
         // default set should always be present as it is part of the
         // distribution (JAR)

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/it/ItalianAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/it/ItalianAnalyzer.java
@@ -75,7 +75,8 @@ private static class DefaultSetHolder {
         DEFAULT_STOP_SET =
             WordlistLoader.getSnowballWordSet(
                 IOUtils.getDecodingReader(
-                    SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
+                    () -> SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                    StandardCharsets.UTF_8));
       } catch (IOException ex) {
         // default set should always be present as it is part of the
         // distribution (JAR)

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/lv/LatvianAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/lv/LatvianAnalyzer.java
@@ -64,7 +64,8 @@ private static class DefaultSetHolder {
         DEFAULT_STOP_SET =
             WordlistLoader.getWordSet(
                 IOUtils.getDecodingReader(
-                    LatvianAnalyzer.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
+                    () -> LatvianAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                    StandardCharsets.UTF_8));
       } catch (IOException ex) {
         // default set should always be present as it is part of the
         // distribution (JAR)

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java
@@ -70,7 +70,8 @@ private static class DefaultSetHolder {
         DEFAULT_STOP_SET =
             WordlistLoader.getSnowballWordSet(
                 IOUtils.getDecodingReader(
-                    SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
+                    () -> SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                    StandardCharsets.UTF_8));
       } catch (IOException ex) {
         // default set should always be present as it is part of the
         // distribution (JAR)

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/no/NorwegianAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/no/NorwegianAnalyzer.java
@@ -66,7 +66,8 @@ private static class DefaultSetHolder {
         DEFAULT_STOP_SET =
             WordlistLoader.getSnowballWordSet(
                 IOUtils.getDecodingReader(
-                    SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
+                    () -> SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                    StandardCharsets.UTF_8));
       } catch (IOException ex) {
         // default set should always be present as it is part of the
         // distribution (JAR)

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseAnalyzer.java
@@ -65,7 +65,8 @@ private static class DefaultSetHolder {
         DEFAULT_STOP_SET =
             WordlistLoader.getSnowballWordSet(
                 IOUtils.getDecodingReader(
-                    SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
+                    () -> SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                    StandardCharsets.UTF_8));
       } catch (IOException ex) {
         // default set should always be present as it is part of the
         // distribution (JAR)

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java
@@ -53,7 +53,8 @@ private static class DefaultSetHolder {
         DEFAULT_STOP_SET =
             WordlistLoader.getSnowballWordSet(
                 IOUtils.getDecodingReader(
-                    SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
+                    () -> SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                    StandardCharsets.UTF_8));
       } catch (IOException ex) {
         // default set should always be present as it is part of the
         // distribution (JAR)

diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishAnalyzer.java
@@ -66,7 +66,8 @@ private static class DefaultSetHolder {
         DEFAULT_STOP_SET =
             WordlistLoader.getSnowballWordSet(
                 IOUtils.getDecodingReader(
-                    SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
+                    () -> SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                    StandardCharsets.UTF_8));
       } catch (IOException ex) {
         // default set should always be present as it is part of the
         // distribution (JAR)

diff --git a/...e/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java b/...e/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java
@@ -87,7 +87,8 @@ static CharArraySet loadDefaultStopWordSet() throws IOException {
       return CharArraySet.unmodifiableSet(
           WordlistLoader.getWordSet(
               IOUtils.getDecodingReader(
-                  SmartChineseAnalyzer.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8),
+                  () -> SmartChineseAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                  StandardCharsets.UTF_8),
               STOPWORD_FILE_COMMENT));
     }
   }

diff --git a/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java b/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java
@@ -76,7 +76,8 @@ private static class DefaultsHolder {
         DEFAULT_STOP_SET =
             WordlistLoader.getWordSet(
                 IOUtils.getDecodingReader(
-                    PolishAnalyzer.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8),
+                    () -> PolishAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
+                    StandardCharsets.UTF_8),
                 "#");
       } catch (IOException ex) {
         // default set should always be present as it is part of the

diff --git a/lucene/core/src/java/org/apache/lucene/util/IOUtils.java b/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
@@ -39,6 +39,7 @@
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.Supplier;
 import org.apache.lucene.store.Directory;
 
 /**
@@ -156,6 +157,27 @@ public static Reader getDecodingReader(InputStream stream, Charset charSet) {
     return new BufferedReader(new InputStreamReader(stream, charSetDecoder));
   }
 
+  /**
+   * Opens a Reader for the stream supplied by the provided {@link Supplier} using a {@link
+   * CharsetDecoder}. Unlike Java's defaults this reader will throw an exception if your it detects
+   * the read charset doesn't match the expected {@link Charset}.
+   *
+   * <p>Decoding readers are useful to load configuration files, stopword lists or synonym files to
+   * detect character set problems. However, it's not recommended to use as a common purpose reader.
+   *
+   * @param streamSupplier A supplier of the input stream for docoding.
+   * @param charSet the expected charset
+   * @return a reader to read the given file
+   */
+  public static Reader getDecodingReader(Supplier<InputStream> streamSupplier, Charset charSet)
+      throws IOException {
+    var is = streamSupplier.get();
+    if (is == null) {
+      throw new IOException("The input stream for decoding must not be null.");
+    }
+    return getDecodingReader(is, charSet);
+  }
+
   /**
    * Opens a Reader for the given resource using a {@link CharsetDecoder}. Unlike Java's defaults
    * this reader will throw an exception if your it detects the read charset doesn't match the
@@ -168,7 +190,10 @@ public static Reader getDecodingReader(InputStream stream, Charset charSet) {
    * @param resource the resource name to load
    * @param charSet the expected charset
    * @return a reader to read the given file
+   * @deprecated This method is caller sensitive and may not work with the module system. Please use
+   *     {@link #getDecodingReader(Supplier, Charset)} instead.
    */
+  @Deprecated
   public static Reader getDecodingReader(Class<?> clazz, String resource, Charset charSet)
       throws IOException {
     InputStream stream = null;

diff --git a/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java b/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
@@ -732,8 +732,9 @@ static int findSingleQuoteStringEnd(String text, int start) {
       final Properties props = new Properties();
       try (Reader in =
           IOUtils.getDecodingReader(
-              JavascriptCompiler.class,
-              JavascriptCompiler.class.getSimpleName() + ".properties",
+              () ->
+                  JavascriptCompiler.class.getResourceAsStream(
+                      JavascriptCompiler.class.getSimpleName() + ".properties"),
               StandardCharsets.UTF_8)) {
         props.load(in);
       }