SONAR-7653 upgrade to Elasticsearch 2.3.3

SonarSource · May 31, 2016 · 05266f9 · 05266f9
1 parent 7767037
commit 05266f9
Show file tree

Hide file tree

Showing 108 changed files with 954 additions and 878 deletions.
diff --git a/pom.xml b/pom.xml
@@ -65,7 +65,7 @@
     <logback.version>1.1.3</logback.version>
     <slf4j.version>1.7.12</slf4j.version>
     <tomcat.version>8.0.30</tomcat.version>
-    <elasticsearch.version>1.7.5</elasticsearch.version>
+    <elasticsearch.version>2.3.3</elasticsearch.version>
     <orchestrator.version>3.11</orchestrator.version>
     <okhttp.version>2.6.0</okhttp.version>
 
@@ -554,32 +554,15 @@
         <version>${sonarCSharp.version}</version>
         <type>sonar-plugin</type>
       </dependency>
+      <dependency>
+        <groupId>net.java.dev.jna</groupId>
+        <artifactId>jna</artifactId>
+        <version>4.1.0</version>
+      </dependency>
       <dependency>
         <groupId>org.elasticsearch</groupId>
         <artifactId>elasticsearch</artifactId>
         <version>${elasticsearch.version}</version>
-        <exclusions>
-          <exclusion>
-            <groupId>org.apache.lucene</groupId>
-            <artifactId>lucene-spatial</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.ow2.asm</groupId>
-            <artifactId>asm</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.ow2.asm</groupId>
-            <artifactId>asm-commons</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.antlr</groupId>
-            <artifactId>antlr-runtime</artifactId>
-          </exclusion>
-          <exclusion>
-            <groupId>org.yaml</groupId>
-            <artifactId>snakeyaml</artifactId>
-          </exclusion>
-        </exclusions>
       </dependency>
       <dependency>
         <groupId>com.github.tlrx</groupId>

diff --git a/server/sonar-search/pom.xml b/server/sonar-search/pom.xml
@@ -23,6 +23,10 @@
       <groupId>org.elasticsearch</groupId>
       <artifactId>elasticsearch</artifactId>
     </dependency>
+    <dependency>
+      <groupId>net.java.dev.jna</groupId>
+      <artifactId>jna</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.google.code.findbugs</groupId>
       <artifactId>jsr305</artifactId>

diff --git a/server/sonar-search/src/main/java/org/sonar/search/EsSettings.java b/server/sonar-search/src/main/java/org/sonar/search/EsSettings.java
@@ -20,13 +20,14 @@
 package org.sonar.search;
 
 import java.io.File;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.util.Arrays;
 import java.util.LinkedHashSet;
 import java.util.Set;
 import java.util.TreeSet;
 import org.apache.commons.lang.StringUtils;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -72,7 +73,7 @@ public String getNodeName() {
   }
 
   Settings build() {
-    ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder();
+    Settings.Builder builder = Settings.builder();
     configureFileSystem(builder);
     configureIndexDefaults(builder);
     configureNetwork(builder);
@@ -81,10 +82,9 @@ Settings build() {
     return builder.build();
   }
 
-  private void configureFileSystem(ImmutableSettings.Builder builder) {
+  private void configureFileSystem(Settings.Builder builder) {
     File homeDir = props.nonNullValueAsFile(ProcessProperties.PATH_HOME);
     File dataDir;
-    File workDir;
     File logDir;
 
     // data dir
@@ -96,15 +96,8 @@ private void configureFileSystem(ImmutableSettings.Builder builder) {
     }
     builder.put("path.data", dataDir.getAbsolutePath());
 
-    // working dir
-    String workPath = props.value(ProcessProperties.PATH_TEMP);
-    if (StringUtils.isNotEmpty(workPath)) {
-      workDir = new File(workPath);
-    } else {
-      workDir = new File(homeDir, "temp");
-    }
-    builder.put("path.work", workDir.getAbsolutePath());
-    builder.put("path.plugins", workDir.getAbsolutePath());
+    String tempPath = props.value(ProcessProperties.PATH_TEMP);
+    builder.put("path.home", new File(tempPath, "es"));
 
     // log dir
     String logPath = props.value(ProcessProperties.PATH_LOGS);
@@ -116,16 +109,16 @@ private void configureFileSystem(ImmutableSettings.Builder builder) {
     builder.put("path.logs", logDir.getAbsolutePath());
   }
 
-  private void configureNetwork(ImmutableSettings.Builder builder) {
-    // the following properties can't be null as default values are defined by app process
-    String host = props.nonNullValue(ProcessProperties.SEARCH_HOST);
+  private void configureNetwork(Settings.Builder builder) {
+    InetAddress host = readHost();
     int port = Integer.parseInt(props.nonNullValue(ProcessProperties.SEARCH_PORT));
     LOGGER.info("Elasticsearch listening on {}:{}", host, port);
 
     // disable multicast
     builder.put("discovery.zen.ping.multicast.enabled", "false");
     builder.put("transport.tcp.port", port);
-    builder.put("transport.host", host);
+    builder.put("transport.host", host.getHostAddress());
+    builder.put("network.host", host.getHostAddress());
 
     // Elasticsearch sets the default value of TCP reuse address to true only on non-MSWindows machines, but why ?
     builder.put("network.tcp.reuse_address", true);
@@ -140,20 +133,29 @@ private void configureNetwork(ImmutableSettings.Builder builder) {
       // see https://github.com/lmenezes/elasticsearch-kopf/issues/195
       builder.put("http.cors.enabled", true);
       builder.put("http.enabled", true);
-      builder.put("http.host", host);
+      builder.put("http.host", host.getHostAddress());
       builder.put("http.port", httpPort);
     }
   }
 
-  private static void configureIndexDefaults(ImmutableSettings.Builder builder) {
+  private InetAddress readHost() {
+    String hostProperty = props.nonNullValue(ProcessProperties.SEARCH_HOST);
+    try {
+      return InetAddress.getByName(hostProperty);
+    } catch (UnknownHostException e) {
+      throw new IllegalStateException("Can not resolve host [" + hostProperty + "]. Please check network settings and property " + ProcessProperties.SEARCH_HOST, e);
+    }
+  }
+
+  private static void configureIndexDefaults(Settings.Builder builder) {
     builder
       .put("index.number_of_shards", "1")
       .put("index.refresh_interval", "30s")
       .put("action.auto_create_index", false)
       .put("index.mapper.dynamic", false);
   }
 
-  private void configureCluster(ImmutableSettings.Builder builder) {
+  private void configureCluster(Settings.Builder builder) {
     int replicationFactor = 0;
     if (inCluster()) {
       replicationFactor = 1;
@@ -178,7 +180,7 @@ private void configureCluster(ImmutableSettings.Builder builder) {
     builder.put("node.name", nodeName);
   }
 
-  private void configureMarvel(ImmutableSettings.Builder builder) {
+  private void configureMarvel(Settings.Builder builder) {
     Set<String> marvels = new TreeSet<>();
     marvels.addAll(Arrays.asList(StringUtils.split(props.value(PROP_MARVEL_HOSTS, ""), ",")));
 

diff --git a/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java b/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java
@@ -19,9 +19,11 @@
  */
 package org.sonar.search;
 
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
+import org.apache.lucene.util.StringHelper;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.node.internal.InternalNode;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.node.NodeBuilder;
 import org.sonar.process.Jmx;
 import org.sonar.process.MinimumViableSystem;
 import org.sonar.process.Monitored;
@@ -31,7 +33,7 @@
 public class SearchServer implements Monitored {
 
   private final EsSettings settings;
-  private InternalNode node;
+  private Node node;
 
   public SearchServer(Props props) {
     this.settings = new EsSettings(props);
@@ -42,10 +44,17 @@ public SearchServer(Props props) {
   @Override
   public void start() {
     Jmx.register(EsSettingsMBean.OBJECT_NAME, settings);
-    node = new InternalNode(settings.build(), false);
+    initBootstrap();
+    node = NodeBuilder.nodeBuilder().settings(settings.build()).build();
     node.start();
   }
 
+  // copied from https://github.com/elastic/elasticsearch/blob/v2.3.3/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
+  private void initBootstrap() {
+    // init lucene random seed. it will use /dev/urandom where available:
+    StringHelper.randomId();
+  }
+
   @Override
   public boolean isUp() {
     return node != null && node.client().admin().cluster().prepareHealth()

diff --git a/server/sonar-search/src/test/java/org/sonar/search/EsSettingsTest.java b/server/sonar-search/src/test/java/org/sonar/search/EsSettingsTest.java
@@ -19,6 +19,9 @@
  */
 package org.sonar.search;
 
+import java.io.File;
+import java.io.IOException;
+import java.util.Properties;
 import org.elasticsearch.common.settings.Settings;
 import org.junit.Rule;
 import org.junit.Test;
@@ -27,10 +30,6 @@
 import org.sonar.process.ProcessProperties;
 import org.sonar.process.Props;
 
-import java.io.File;
-import java.io.IOException;
-import java.util.Properties;
-
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.fail;
 
@@ -60,7 +59,7 @@ public void test_default_settings() throws Exception {
 
     assertThat(generated.get("path.data")).isNotNull();
     assertThat(generated.get("path.logs")).isNotNull();
-    assertThat(generated.get("path.work")).isNotNull();
+    assertThat(generated.get("path.home")).isNotNull();
 
     // http is disabled for security reasons
     assertThat(generated.get("http.enabled")).isEqualTo("false");
@@ -98,7 +97,7 @@ public void override_dirs() throws Exception {
 
     assertThat(settings.get("path.data")).isEqualTo(new File(dataDir, "es").getAbsolutePath());
     assertThat(settings.get("path.logs")).isEqualTo(logDir.getAbsolutePath());
-    assertThat(settings.get("path.work")).isEqualTo(tempDir.getAbsolutePath());
+    assertThat(settings.get("path.home")).isEqualTo(new File(tempDir, "es").getAbsolutePath());
   }
 
   @Test

diff --git a/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java b/server/sonar-search/src/test/java/org/sonar/search/SearchServerTest.java
@@ -21,11 +21,10 @@
 
 import java.net.InetAddress;
 import java.util.Properties;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.junit.After;
@@ -72,8 +71,8 @@ public void tearDown() {
   public void start_stop_server() throws Exception {
     Props props = new Props(new Properties());
     // the following properties have always default values (see ProcessProperties)
-    String host = InetAddress.getLocalHost().getHostAddress();
-    props.set(ProcessProperties.SEARCH_HOST, host);
+    InetAddress host = InetAddress.getLocalHost();
+    props.set(ProcessProperties.SEARCH_HOST, host.getHostAddress());
     props.set(ProcessProperties.SEARCH_PORT, String.valueOf(port));
     props.set(ProcessProperties.CLUSTER_NAME, CLUSTER_NAME);
     props.set(ProcessProperties.CLUSTER_NODE_NAME, "test");
@@ -84,8 +83,8 @@ public void start_stop_server() throws Exception {
     searchServer.start();
     assertThat(searchServer.isUp()).isTrue();
 
-    Settings settings = ImmutableSettings.settingsBuilder().put("cluster.name", CLUSTER_NAME).build();
-    client = new TransportClient(settings)
+    Settings settings = Settings.builder().put("cluster.name", CLUSTER_NAME).build();
+    client = TransportClient.builder().settings(settings).build()
       .addTransportAddress(new InetSocketTransportAddress(host, port));
     assertThat(client.admin().cluster().prepareClusterStats().get().getStatus()).isEqualTo(ClusterHealthStatus.GREEN);
 

diff --git a/...r-server-benchmarks/src/test/java/org/sonar/server/benchmark/IssueIndexBenchmarkTest.java b/...r-server-benchmarks/src/test/java/org/sonar/server/benchmark/IssueIndexBenchmarkTest.java
@@ -116,7 +116,7 @@ private void benchmarkIssueIndexing() {
     benchmark.expectAround("Throughput to index issues", throughputPerSecond, 6500, Benchmark.DEFAULT_ERROR_MARGIN_PERCENTS);
 
     // be sure that physical files do not evolve during estimation of size
-    tester.get(EsClient.class).prepareOptimize("issues").get();
+    tester.get(EsClient.class).prepareForceMerge("issues").get();
     long dirSize = FileUtils.sizeOfDirectory(tester.getEsServerHolder().getHomeDir());
     LOGGER.info(String.format("ES dir: " + FileUtils.byteCountToDisplaySize(dirSize)));
     benchmark.expectBetween("ES dir size (b)", dirSize, 200L * FileUtils.ONE_MB, 420L * FileUtils.ONE_MB);

diff --git a/server/sonar-server/pom.xml b/server/sonar-server/pom.xml
@@ -159,6 +159,17 @@
       <groupId>org.elasticsearch</groupId>
       <artifactId>elasticsearch</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.elasticsearch</groupId>
+      <artifactId>elasticsearch</artifactId>
+      <version>${elasticsearch.version}</version>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
+      <groupId>net.java.dev.jna</groupId>
+      <artifactId>jna</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>

diff --git a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityDoc.java b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityDoc.java
@@ -39,6 +39,21 @@ public ActivityDoc(Map<String, Object> fields) {
     super(new HashMap<String, Object>());
   }
 
+  @Override
+  public String getId() {
+    return getKey();
+  }
+
+  @Override
+  public String getRouting() {
+    return null;
+  }
+
+  @Override
+  public String getParent() {
+    return null;
+  }
+
   public void setCreatedAt(Date date) {
     setField(ActivityIndexDefinition.FIELD_CREATED_AT, date);
   }