SONAR-8798 compile with ES 5.0.0

pom.xml

@@ -75,7 +75,7 @@
     <logback.version>1.1.7</logback.version>
     <slf4j.version>1.7.21</slf4j.version>
     <tomcat.version>8.5.16</tomcat.version>
-    <elasticsearch.version>2.4.4</elasticsearch.version>
+    <elasticsearch.version>5.0.0</elasticsearch.version>
     <orchestrator.version>3.15.0.1090</orchestrator.version>
     <okhttp.version>3.7.0</okhttp.version>
     <jackson.version>2.6.6</jackson.version>
@@ -842,6 +842,11 @@
         <artifactId>log4j-over-slf4j</artifactId>
         <version>${slf4j.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.logging.log4j</groupId>
+        <artifactId>log4j-api</artifactId>
+        <version>2.7</version>
+      </dependency>
       <dependency>
         <groupId>ch.qos.logback</groupId>
         <artifactId>logback-access</artifactId>

server/sonar-search/pom.xml

@@ -35,16 +35,14 @@
 
 
     <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-api</artifactId>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-api</artifactId>
+      <version>2.6.2</version>
     </dependency>
     <dependency>
-      <groupId>ch.qos.logback</groupId>
-      <artifactId>logback-classic</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>ch.qos.logback</groupId>
-      <artifactId>logback-core</artifactId>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-core</artifactId>
+      <version>2.6.2</version>
     </dependency>
 
     <!-- testing -->

server/sonar-search/src/main/java/org/sonar/search/SearchServer.java

@@ -19,12 +19,13 @@
  */
 package org.sonar.search;
 
+import java.io.IOException;
 import org.apache.lucene.util.StringHelper;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.node.NodeBuilder;
+import org.elasticsearch.node.NodeValidationException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.process.Jmx;
@@ -56,11 +57,14 @@ public void start() {
     if (esSettings.getAsInt(MIMINUM_MASTER_NODES, 1) >= 2) {
       LOGGER.info("Elasticsearch is waiting {} for {} node(s) to be up to start.",
         esSettings.get(INITIAL_STATE_TIMEOUT),
-        esSettings.get(MIMINUM_MASTER_NODES)
-      );
+        esSettings.get(MIMINUM_MASTER_NODES));
+    }
+    node = new Node(settings.build());
+    try {
+      node.start();
+    } catch (NodeValidationException e) {
+      throw new RuntimeException("Failed to start ES", e);
     }
-    node = NodeBuilder.nodeBuilder().settings(esSettings).build();
-    node.start();
   }
 
   // copied from https://github.com/elastic/elasticsearch/blob/v2.3.3/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@@ -72,10 +76,10 @@ private static void initBootstrap() {
   @Override
   public Status getStatus() {
     boolean esStatus = node != null && node.client().admin().cluster().prepareHealth()
-        .setWaitForYellowStatus()
-        .setTimeout(TimeValue.timeValueSeconds(30L))
-        .get()
-        .getStatus() != ClusterHealthStatus.RED;
+      .setWaitForYellowStatus()
+      .setTimeout(TimeValue.timeValueSeconds(30L))
+      .get()
+      .getStatus() != ClusterHealthStatus.RED;
     if (esStatus) {
       return Status.OPERATIONAL;
     }
@@ -97,7 +101,11 @@ public void awaitStop() {
   @Override
   public void stop() {
     if (node != null && !node.isClosed()) {
-      node.close();
+      try {
+        node.close();
+      } catch (IOException e) {
+        LOGGER.error("Failed to stop ES cleanly", e);
+      }
     }
     Jmx.unregister(EsSettingsMBean.OBJECT_NAME);
   }

server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java

@@ -21,13 +21,16 @@
 
 import java.io.IOException;
 import java.net.InetAddress;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.Properties;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.plugins.Plugin;
 import org.junit.After;
 import org.junit.Rule;
 import org.junit.Test;
@@ -86,6 +89,7 @@ public void log_information_on_startup() throws IOException {
     Logger logger = mock(Logger.class);
     underTest.LOGGER = logger;
     underTest.start();
+    System.out.println(org.apache.logging.log4j.Logger.class);
     verify(logger).info(eq("Elasticsearch is waiting {} for {} node(s) to be up to start."), eq("1s"), eq("2"));
   }
 
@@ -109,8 +113,10 @@ public void start_stop_server() throws Exception {
     assertThat(underTest.getStatus()).isEqualTo(Monitored.Status.OPERATIONAL);
 
     Settings settings = Settings.builder().put("cluster.name", A_CLUSTER_NAME).build();
-    client = TransportClient.builder().settings(settings).build()
-      .addTransportAddress(new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), port));
+    Collection<Class<? extends Plugin>> plugins = Collections.emptyList();
+    client = new TransportClient(settings, plugins) {
+
+    }.addTransportAddress(new InetSocketTransportAddress(InetAddress.getLoopbackAddress(), port));
     assertThat(client.admin().cluster().prepareClusterStats().get().getStatus()).isEqualTo(ClusterHealthStatus.GREEN);
 
     underTest.stop();

server/sonar-server/pom.xml

@@ -27,6 +27,10 @@
       <groupId>org.slf4j</groupId>
       <artifactId>jul-to-slf4j</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-api</artifactId>
+    </dependency>
     <dependency>
       <groupId>ch.qos.logback</groupId>
       <artifactId>logback-access</artifactId>
@@ -148,11 +152,10 @@
       <artifactId>elasticsearch</artifactId>
     </dependency>
     <dependency>
-      <groupId>org.elasticsearch</groupId>
-      <artifactId>elasticsearch</artifactId>
+      <groupId>org.elasticsearch.test</groupId>
+      <artifactId>framework</artifactId>
       <version>${elasticsearch.version}</version>
       <scope>test</scope>
-      <type>test-jar</type>
     </dependency>
     <dependency>
       <groupId>net.java.dev.jna</groupId>

server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java

@@ -33,10 +33,10 @@
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.filters.InternalFilters;
-import org.elasticsearch.search.aggregations.bucket.filters.InternalFilters.Bucket;
+import org.elasticsearch.search.aggregations.bucket.filters.InternalFilters.InternalBucket;
 import org.elasticsearch.search.aggregations.metrics.tophits.InternalTopHits;
-import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsBuilder;
-import org.elasticsearch.search.highlight.HighlightBuilder;
+import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder;
+import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.ScoreSortBuilder;
 import org.sonar.server.es.EsClient;
@@ -67,7 +67,7 @@ public ComponentIndex(EsClient client, AuthorizationTypeSupport authorizationTyp
     this.authorizationTypeSupport = authorizationTypeSupport;
   }
 
-  private static HighlightBuilder.Field createHighlighter() {
+  private static HighlightBuilder.Field createHighlighterField() {
     HighlightBuilder.Field field = new HighlightBuilder.Field(FIELD_NAME);
     field.highlighterType("fvh");
     field.matchedFields(
@@ -105,23 +105,25 @@ ComponentIndexResults search(ComponentIndexQuery query, ComponentTextSearchFeatu
   }
 
   private static FiltersAggregationBuilder createAggregation(ComponentIndexQuery query) {
-    FiltersAggregationBuilder filtersAggregation = AggregationBuilders.filters(FILTERS_AGGREGATION_NAME)
+    return AggregationBuilders.filters(
+      FILTERS_AGGREGATION_NAME,
+      query.getQualifiers().stream().map(q -> termQuery(FIELD_QUALIFIER, q)).toArray(QueryBuilder[]::new))
       .subAggregation(createSubAggregation(query));
-    query.getQualifiers().forEach(q -> filtersAggregation.filter(q, termQuery(FIELD_QUALIFIER, q)));
-    return filtersAggregation;
   }
 
-  private static TopHitsBuilder createSubAggregation(ComponentIndexQuery query) {
-    TopHitsBuilder sub = AggregationBuilders.topHits(DOCS_AGGREGATION_NAME)
-      .setHighlighterEncoder("html")
-      .setHighlighterPreTags("<mark>")
-      .setHighlighterPostTags("</mark>")
-      .addHighlightedField(createHighlighter())
-      .setFrom(query.getSkip())
-      .setSize(query.getLimit())
-      .addSort(new ScoreSortBuilder())
-      .addSort(new FieldSortBuilder(ComponentIndexDefinition.FIELD_NAME));
-    return sub.setFetchSource(false);
+  private static TopHitsAggregationBuilder createSubAggregation(ComponentIndexQuery query) {
+    return AggregationBuilders.topHits(DOCS_AGGREGATION_NAME)
+      .highlighter(new HighlightBuilder()
+        .encoder("html")
+        .preTags("<mark>")
+        .postTags("</mark>")
+        .field(createHighlighterField())
+      )
+      .from(query.getSkip())
+      .size(query.getLimit())
+      .sort(new ScoreSortBuilder())
+      .sort(new FieldSortBuilder(ComponentIndexDefinition.FIELD_NAME))
+      .fetchSource(false);
   }
 
   private QueryBuilder createQuery(ComponentIndexQuery query, ComponentTextSearchFeature... features) {
@@ -139,14 +141,14 @@ private QueryBuilder createQuery(ComponentIndexQuery query, ComponentTextSearchF
 
   private static ComponentIndexResults aggregationsToQualifiers(SearchResponse response) {
     InternalFilters filtersAgg = response.getAggregations().get(FILTERS_AGGREGATION_NAME);
-    List<Bucket> buckets = filtersAgg.getBuckets();
+    List<InternalBucket> buckets = filtersAgg.getBuckets();
     return ComponentIndexResults.newBuilder()
       .setQualifiers(
         buckets.stream().map(ComponentIndex::bucketToQualifier))
       .build();
   }
 
-  private static ComponentHitsPerQualifier bucketToQualifier(Bucket bucket) {
+  private static ComponentHitsPerQualifier bucketToQualifier(InternalBucket bucket) {
     InternalTopHits docs = bucket.getAggregations().get(DOCS_AGGREGATION_NAME);
 
     SearchHits hitList = docs.getHits();

server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java

@@ -20,6 +20,7 @@
 package org.sonar.server.es;
 
 import java.util.Arrays;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 
@@ -43,18 +44,17 @@ public class DefaultIndexSettings {
 
   public static final String TYPE = "type";
   public static final String INDEX = "index";
-  public static final String INDEX_SEARCHABLE_FOR_TEXT = "analyzed"; // ES 5: remove this constant and replace usages with INDEX_SEARCHABLE
-  public static final String INDEX_SEARCHABLE_FOR_KEYWORD = "not_analyzed"; // ES 5: change to true and rename to INDEX_SEARCHABLE
-  public static final String INDEX_NOT_SEARCHABLE = "no"; // ES 5: change to false
-  public static final String FIELD_TYPE_TEXT = "string";  // ES 5: change to text
-  public static final String FIELD_TYPE_KEYWORD = "string";  // ES 5: change to keyword
+  public static final String INDEX_SEARCHABLE = "true";
+  public static final String INDEX_NOT_SEARCHABLE = "false";
+  public static final String FIELD_TYPE_TEXT = "text";
+  public static final String FIELD_TYPE_KEYWORD = "keyword";
   public static final String STANDARD = "standard";
   public static final String PATTERN = "pattern";
   public static final String CUSTOM = "custom";
   public static final String KEYWORD = "keyword";
   public static final String CLASSIC = "classic";
-  public static final boolean REFRESH_IMMEDIATE = true; // ES 5: replace with org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE
-  public static final boolean REFRESH_NONE = false; // ES 5: replace with org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE
+  public static final RefreshPolicy REFRESH_IMMEDIATE = RefreshPolicy.IMMEDIATE;
+  public static final RefreshPolicy REFRESH_NONE = RefreshPolicy.NONE;
 
   public static final String TRUNCATE = "truncate";
 

server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettingsElement.java

@@ -33,7 +33,7 @@
 import static org.sonar.server.es.DefaultIndexSettings.FIELD_TYPE_TEXT;
 import static org.sonar.server.es.DefaultIndexSettings.FILTER;
 import static org.sonar.server.es.DefaultIndexSettings.INDEX;
-import static org.sonar.server.es.DefaultIndexSettings.INDEX_SEARCHABLE_FOR_TEXT;
+import static org.sonar.server.es.DefaultIndexSettings.INDEX_SEARCHABLE;
 import static org.sonar.server.es.DefaultIndexSettings.KEYWORD;
 import static org.sonar.server.es.DefaultIndexSettings.LOWERCASE;
 import static org.sonar.server.es.DefaultIndexSettings.MAXIMUM_NGRAM_LENGTH;
@@ -125,7 +125,7 @@ protected void setup() {
     public SortedMap<String, String> fieldMapping() {
       return ImmutableSortedMap.of(
         TYPE, FIELD_TYPE_TEXT,
-        INDEX, INDEX_SEARCHABLE_FOR_TEXT,
+        INDEX, INDEX_SEARCHABLE,
         ANALYZER, getName());
     }
   },
@@ -149,7 +149,7 @@ protected void setup() {
     public SortedMap<String, String> fieldMapping() {
       return ImmutableSortedMap.of(
         TYPE, FIELD_TYPE_TEXT,
-        INDEX, INDEX_SEARCHABLE_FOR_TEXT,
+        INDEX, INDEX_SEARCHABLE,
         ANALYZER, INDEX_GRAMS_ANALYZER.getName(),
         SEARCH_ANALYZER, getName());
     }
@@ -174,7 +174,7 @@ protected void setup() {
     public SortedMap<String, String> fieldMapping() {
       return ImmutableSortedMap.of(
         TYPE, FIELD_TYPE_TEXT,
-        INDEX, INDEX_SEARCHABLE_FOR_TEXT,
+        INDEX, INDEX_SEARCHABLE,
         ANALYZER, INDEX_PREFIX_ANALYZER.getName(),
         SEARCH_ANALYZER, getName());
     }
@@ -199,7 +199,7 @@ protected void setup() {
     public SortedMap<String, String> fieldMapping() {
       return ImmutableSortedMap.of(
         TYPE, FIELD_TYPE_TEXT,
-        INDEX, INDEX_SEARCHABLE_FOR_TEXT,
+        INDEX, INDEX_SEARCHABLE,
         ANALYZER, INDEX_PREFIX_CASE_INSENSITIVE_ANALYZER.getName(),
         SEARCH_ANALYZER, getName());
     }
@@ -224,7 +224,7 @@ protected void setup() {
     public SortedMap<String, String> fieldMapping() {
       return ImmutableSortedMap.of(
         TYPE, FIELD_TYPE_TEXT,
-        INDEX, INDEX_SEARCHABLE_FOR_TEXT,
+        INDEX, INDEX_SEARCHABLE,
         ANALYZER, USER_INDEX_GRAMS_ANALYZER.getName(),
         SEARCH_ANALYZER, getName());
     }
@@ -249,7 +249,7 @@ protected void setup() {
     public SortedMap<String, String> fieldMapping() {
       return ImmutableSortedMap.of(
         TYPE, FIELD_TYPE_TEXT,
-        INDEX, INDEX_SEARCHABLE_FOR_TEXT,
+        INDEX, INDEX_SEARCHABLE,
         ANALYZER, INDEX_WORDS_ANALYZER.getName(),
         SEARCH_ANALYZER, getName());
     }

server/sonar-server/src/main/java/org/sonar/server/es/EsClientProvider.java

@@ -23,6 +23,7 @@
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.stream.Collectors;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
@@ -54,13 +55,13 @@ public EsClient provide(Configuration config) {
       boolean clusterEnabled = config.getBoolean(ProcessProperties.CLUSTER_ENABLED).orElse(false);
       if (clusterEnabled && config.getBoolean(ProcessProperties.CLUSTER_SEARCH_DISABLED).orElse(false)) {
         esSettings.put("client.transport.sniff", true);
-        nativeClient = TransportClient.builder().settings(esSettings).build();
+        nativeClient = new TransportClient(esSettings.build(), Collections.emptyList()) {};
         Arrays.stream(config.getStringArray(ProcessProperties.CLUSTER_SEARCH_HOSTS))
           .map(HostAndPort::fromString)
           .forEach(h -> addHostToClient(h, nativeClient));
         LOGGER.info("Connected to remote Elasticsearch: [{}]", displayedAddresses(nativeClient));
       } else {
-        nativeClient = TransportClient.builder().settings(esSettings).build();
+        nativeClient = new TransportClient(esSettings.build(), Collections.emptyList()) {};
         HostAndPort host = HostAndPort.fromParts(config.get(ProcessProperties.SEARCH_HOST).get(), config.getInt(ProcessProperties.SEARCH_PORT).get());
         addHostToClient(host, nativeClient);
         LOGGER.info("Connected to local Elasticsearch: [{}]", displayedAddresses(nativeClient));