Fixing JDK-11 and HighlighterSearchIT::testPostingsHighlighter

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

.github/workflows/precommit.yml

@@ -6,12 +6,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      # todo lower back to 11 after figuring out
-      # why precommit fails w/ 11 and lucene 9
-      - name: Set up JDK 14
+      - name: Set up JDK 11
         uses: actions/setup-java@v2
         with:
-          java-version: 14
+          java-version: 11
           distribution: adopt
       - name: Run Gradle
         run: |

build.gradle

@@ -230,7 +230,10 @@ tasks.register("branchConsistency") {
 allprojects {
   // configure compiler options
   tasks.withType(JavaCompile).configureEach { JavaCompile compile ->
-    compile.options.compilerArgs << '-Werror'
+    // See please https://bugs.openjdk.java.net/browse/JDK-8209058
+    if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_11) {
+        compile.options.compilerArgs << '-Werror'
+    }
     compile.options.compilerArgs << '-Xlint:auxiliaryclass'
     compile.options.compilerArgs << '-Xlint:cast'
     compile.options.compilerArgs << '-Xlint:classfile'

buildSrc/src/main/resources/minimumRuntimeVersion

@@ -1 +1 @@
-1.9
+11

server/src/internalClusterTest/java/org/opensearch/search/fetch/subphase/highlight/HighlighterSearchIT.java

@@ -2282,7 +2282,6 @@ public void testHighlightNoMatchSizeNumberOfFragments() throws IOException {
         assertHighlight(response, 0, "text", 1, 2, equalTo("This is the <em>fifth</em> sentence"));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/2063")
     public void testPostingsHighlighter() throws Exception {
         assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping()));
         ensureGreen();
@@ -3288,7 +3287,6 @@ public void testKeywordFieldHighlighting() throws IOException {
         );
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/2063")
     public void testCopyToFields() throws Exception {
         XContentBuilder b = jsonBuilder().startObject().startObject("properties");
         b.startObject("foo");
@@ -3297,7 +3295,8 @@ public void testCopyToFields() throws Exception {
             b.field("copy_to", "foo_copy");
         }
         b.endObject();
-        b.startObject("foo_copy").field("type", "text").endObject();
+        // If field is not stored, it is looked up in source (but source has only 'foo'
+        b.startObject("foo_copy").field("type", "text").field("store", true).endObject();
         b.endObject().endObject();
         prepareCreate("test").addMapping("type", b).get();
 
@@ -3411,7 +3410,6 @@ public void testFiltersFunctionScoreQueryHighlight() throws Exception {
         assertThat(field.getFragments()[0].string(), equalTo("<em>brown</em>"));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/2063")
     public void testHighlightQueryRewriteDatesWithNow() throws Exception {
         assertAcked(
             client().admin()
@@ -3448,7 +3446,6 @@ public void testHighlightQueryRewriteDatesWithNow() throws Exception {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/2063")
     public void testWithNestedQuery() throws Exception {
         String mapping = Strings.toString(
             jsonBuilder().startObject()

server/src/main/java/org/apache/lucene/search/uhighlight/CustomUnifiedHighlighter.java

@@ -43,6 +43,7 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.uhighlight.UnifiedHighlighter.HighlightFlag;
 import org.apache.lucene.util.BytesRef;
 import org.opensearch.common.CheckedSupplier;
 import org.opensearch.common.Nullable;
@@ -53,6 +54,7 @@
 import java.text.BreakIterator;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.Locale;
 import java.util.Set;
 import java.util.function.Predicate;
@@ -259,4 +261,12 @@ protected OffsetSource getOffsetSource(String field) {
         return offsetSource;
     }
 
+    /** Customize the highlighting flags to use by field. */
+    @Override
+    protected Set<HighlightFlag> getFlags(String field) {
+        final Set<HighlightFlag> flags = super.getFlags(field);
+        // Change the defaults introduced by https://issues.apache.org/jira/browse/LUCENE-9431
+        flags.remove(HighlightFlag.WEIGHT_MATCHES);
+        return flags;
+    }
 }

server/src/main/java/org/opensearch/index/cache/bitset/BitsetFilterCache.java

@@ -33,6 +33,7 @@
 package org.opensearch.index.cache.bitset;
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.lucene.index.FilterLeafReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexReaderContext;
 import org.apache.lucene.index.LeafReaderContext;
@@ -147,7 +148,7 @@ public void clear(String reason) {
     }
 
     private BitSet getAndLoadIfNotPresent(final Query query, final LeafReaderContext context) throws ExecutionException {
-        final IndexReader.CacheHelper cacheHelper = context.reader().getCoreCacheHelper();
+        final IndexReader.CacheHelper cacheHelper = FilterLeafReader.unwrap(context.reader()).getCoreCacheHelper();
         if (cacheHelper == null) {
             throw new IllegalArgumentException("Reader " + context.reader() + " does not support caching");
         }