From 8a0b60e14356d4d2b220c5332c0b2017591f8f67 Mon Sep 17 00:00:00 2001 From: Richard Downer Date: Mon, 10 Apr 2017 10:16:13 +0100 Subject: [PATCH 1/5] ElasticSearch: update config for newest version Brings version up to 5.3.0, updates the download location URL, and specifies the required version of Java to be 1.8. --- .../entity/nosql/elasticsearch/ElasticSearchNode.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNode.java b/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNode.java index d972efc73..5934e6b62 100644 --- a/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNode.java +++ b/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNode.java @@ -45,14 +45,14 @@ public interface ElasticSearchNode extends SoftwareProcess, UsesJava, DatastoreMixins.HasDatastoreUrl { @SetFromFlag("version") - ConfigKey SUGGESTED_VERSION = ConfigKeys.newConfigKeyWithDefault(SoftwareProcess.SUGGESTED_VERSION, "1.2.1"); + ConfigKey SUGGESTED_VERSION = ConfigKeys.newConfigKeyWithDefault(SoftwareProcess.SUGGESTED_VERSION, "5.3.0"); @SetFromFlag("archiveNameFormat") ConfigKey ARCHIVE_DIRECTORY_NAME_FORMAT = ConfigKeys.newConfigKeyWithDefault(SoftwareProcess.ARCHIVE_DIRECTORY_NAME_FORMAT, "elasticsearch-%s"); @SetFromFlag("downloadUrl") AttributeSensorAndConfigKey DOWNLOAD_URL = ConfigKeys.newSensorAndConfigKeyWithDefault(SoftwareProcess.DOWNLOAD_URL, - "https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-${version}.tar.gz"); + "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${version}.tar.gz"); @SetFromFlag("dataDir") ConfigKey DATA_DIR = ConfigKeys.newStringConfigKey("elasticsearch.node.data.dir", "Directory for writing data files", null); @@ -90,6 +90,9 @@ public interface ElasticSearchNode extends SoftwareProcess, UsesJava, DatastoreM @SetFromFlag("useHttpMonitoring") ConfigKey USE_HTTP_MONITORING = ConfigKeys.newConfigKey("httpMonitoring.enabled", "HTTP(S) monitoring enabled", Boolean.TRUE); + @SetFromFlag("javaVersionRequired") + ConfigKey JAVA_VERSION_REQUIRED = ConfigKeys.newStringConfigKey("java.version.required", "Java version required", "1.8"); + AttributeSensor NODE_ID = Sensors.newStringSensor("elasticsearch.node.id"); AttributeSensor DOCUMENT_COUNT = Sensors.newIntegerSensor("elasticsearch.node.docs.count"); AttributeSensor STORE_BYTES = Sensors.newIntegerSensor("elasticsearch.node.store.bytes"); From dfcc39e83d9293fd3f8aba85fdaa84743b386f6f Mon Sep 17 00:00:00 2001 From: Richard Downer Date: Mon, 10 Apr 2017 10:17:17 +0100 Subject: [PATCH 2/5] ElasticSearch: fix command line options Newest version changes the command line options. Fixes appropriately. --- .../ElasticSearchNodeSshDriver.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNodeSshDriver.java b/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNodeSshDriver.java index 764ab0ff9..a7d147b70 100644 --- a/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNodeSshDriver.java +++ b/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNodeSshDriver.java @@ -75,14 +75,14 @@ public void launch() { StringBuilder commandBuilder = new StringBuilder() .append(String.format("%s/bin/elasticsearch -d -p %s", getExpandedInstallDir(), pidFile)); if (entity.getConfig(ElasticSearchNode.TEMPLATE_CONFIGURATION_URL) != null) { - commandBuilder.append(" -Des.config=" + Os.mergePaths(getRunDir(), getConfigFile())); + commandBuilder.append(" -Epath.conf=" + Os.mergePaths(getRunDir(), getConfigFile())); } - appendConfigIfPresent(commandBuilder, "es.path.data", ElasticSearchNode.DATA_DIR, Os.mergePaths(getRunDir(), "data")); - appendConfigIfPresent(commandBuilder, "es.path.logs", ElasticSearchNode.LOG_DIR, Os.mergePaths(getRunDir(), "logs")); - appendConfigIfPresent(commandBuilder, "es.node.name", ElasticSearchNode.NODE_NAME.getConfigKey()); - appendConfigIfPresent(commandBuilder, "es.cluster.name", ElasticSearchNode.CLUSTER_NAME.getConfigKey()); - appendConfigIfPresent(commandBuilder, "es.discovery.zen.ping.multicast.enabled", ElasticSearchNode.MULTICAST_ENABLED); - appendConfigIfPresent(commandBuilder, "es.discovery.zen.ping.unicast.enabled", ElasticSearchNode.UNICAST_ENABLED); + appendConfigIfPresent(commandBuilder, "path.data", ElasticSearchNode.DATA_DIR, Os.mergePaths(getRunDir(), "data")); + appendConfigIfPresent(commandBuilder, "path.logs", ElasticSearchNode.LOG_DIR, Os.mergePaths(getRunDir(), "logs")); + appendConfigIfPresent(commandBuilder, "node.name", ElasticSearchNode.NODE_NAME.getConfigKey()); + appendConfigIfPresent(commandBuilder, "cluster.name", ElasticSearchNode.CLUSTER_NAME.getConfigKey()); + appendConfigIfPresent(commandBuilder, "discovery.zen.ping.multicast.enabled", ElasticSearchNode.MULTICAST_ENABLED); + appendConfigIfPresent(commandBuilder, "discovery.zen.ping.unicast.enabled", ElasticSearchNode.UNICAST_ENABLED); commandBuilder.append(" > out.log 2> err.log < /dev/null"); newScript(MutableMap.of("usePidFile", false), LAUNCHING) .updateTaskAndFailOnNonZeroResultCode() @@ -103,7 +103,7 @@ private void appendConfigIfPresent(StringBuilder builder, String parameter, Conf config = defaultValue; } if (config != null) { - builder.append(String.format(" -D%s=%s", parameter, config)); + builder.append(String.format(" -E%s=%s", parameter, config)); } } From be5d101a3c35cab688330b74829f4db08edcf229 Mon Sep 17 00:00:00 2001 From: Richard Downer Date: Mon, 10 Apr 2017 10:18:28 +0100 Subject: [PATCH 3/5] ElasticSearch: configure to listen on network i/f Newest version defaults to listening on localhost only. Explicitly configure listening on the network. --- .../entity/nosql/elasticsearch/ElasticSearchNodeSshDriver.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNodeSshDriver.java b/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNodeSshDriver.java index a7d147b70..c68ca4b55 100644 --- a/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNodeSshDriver.java +++ b/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNodeSshDriver.java @@ -27,6 +27,7 @@ import org.apache.brooklyn.config.ConfigKey; import org.apache.brooklyn.entity.java.JavaSoftwareProcessSshDriver; import org.apache.brooklyn.entity.software.base.AbstractSoftwareProcessSshDriver; +import org.apache.brooklyn.entity.software.base.SoftwareProcess; import org.apache.brooklyn.location.ssh.SshMachineLocation; import org.apache.brooklyn.util.collections.MutableMap; import org.apache.brooklyn.util.net.Urls; @@ -74,6 +75,7 @@ public void launch() { entity.sensors().set(ElasticSearchNode.PID_FILE, pidFile); StringBuilder commandBuilder = new StringBuilder() .append(String.format("%s/bin/elasticsearch -d -p %s", getExpandedInstallDir(), pidFile)); + commandBuilder.append(" -Enetwork.host=" + getEntity().sensors().get(SoftwareProcess.ADDRESS)); if (entity.getConfig(ElasticSearchNode.TEMPLATE_CONFIGURATION_URL) != null) { commandBuilder.append(" -Epath.conf=" + Os.mergePaths(getRunDir(), getConfigFile())); } From 87b6ff852be73d57121cb51a649d8f0e03c35060 Mon Sep 17 00:00:00 2001 From: Richard Downer Date: Mon, 10 Apr 2017 10:19:46 +0100 Subject: [PATCH 4/5] ElasticSearch: ulimit/sysctl configuration updates Newest version requires increased ulimit/sysctl values. Configure Linux with the required values. --- .../ElasticSearchNodeSshDriver.java | 6 +++++ .../99-elasticsearch-limits.conf | 24 +++++++++++++++++++ .../99-elasticsearch-sysctl.conf | 21 ++++++++++++++++ 3 files changed, 51 insertions(+) create mode 100644 software/nosql/src/main/resources/org/apache/brooklyn/entity/nosql/elasticsearch/99-elasticsearch-limits.conf create mode 100644 software/nosql/src/main/resources/org/apache/brooklyn/entity/nosql/elasticsearch/99-elasticsearch-sysctl.conf diff --git a/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNodeSshDriver.java b/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNodeSshDriver.java index c68ca4b55..eff68ca9b 100644 --- a/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNodeSshDriver.java +++ b/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNodeSshDriver.java @@ -42,10 +42,16 @@ public ElasticSearchNodeSshDriver(ElasticSearchNodeImpl entity, SshMachineLocati @Override public void install() { + final String tmpLimitsFile = Os.mergePaths(getRunDir(), "99-elasticsearch-limits.conf"); + final String tmpSysctlFile = Os.mergePaths(getRunDir(), "99-elasticsearch-sysctl.conf"); + copyResource("classpath://org/apache/brooklyn/entity/nosql/elasticsearch/99-elasticsearch-limits.conf", tmpLimitsFile); + copyResource("classpath://org/apache/brooklyn/entity/nosql/elasticsearch/99-elasticsearch-sysctl.conf", tmpSysctlFile); List urls = resolver.getTargets(); String saveAs = resolver.getFilename(); List commands = ImmutableList.builder() + .add(BashCommands.sudo(String.format("cp %s %s", tmpLimitsFile, "/etc/security/limits.d/99-elasticsearch.conf"))) + .add(BashCommands.sudo(String.format("cp %s %s", tmpSysctlFile, "/etc/sysctl.d/99-elasticsearch.conf"))) .addAll(BashCommands.commandsToDownloadUrlsAs(urls, saveAs)) .add(String.format("tar zxvf %s", saveAs)) .build(); diff --git a/software/nosql/src/main/resources/org/apache/brooklyn/entity/nosql/elasticsearch/99-elasticsearch-limits.conf b/software/nosql/src/main/resources/org/apache/brooklyn/entity/nosql/elasticsearch/99-elasticsearch-limits.conf new file mode 100644 index 000000000..ce0183321 --- /dev/null +++ b/software/nosql/src/main/resources/org/apache/brooklyn/entity/nosql/elasticsearch/99-elasticsearch-limits.conf @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# This file is from Apache Brooklyn: https://brooklyn.apache.org + +* soft nofile 65536 +* hard nofile 65536 +root soft nofile 65536 +root hard nofile 65536 diff --git a/software/nosql/src/main/resources/org/apache/brooklyn/entity/nosql/elasticsearch/99-elasticsearch-sysctl.conf b/software/nosql/src/main/resources/org/apache/brooklyn/entity/nosql/elasticsearch/99-elasticsearch-sysctl.conf new file mode 100644 index 000000000..7fd99ce27 --- /dev/null +++ b/software/nosql/src/main/resources/org/apache/brooklyn/entity/nosql/elasticsearch/99-elasticsearch-sysctl.conf @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# This file is from Apache Brooklyn: https://brooklyn.apache.org + +vm.max_map_count=262144 From 31fac60d85d12e90b06553e622c5d38a0292f760 Mon Sep 17 00:00:00 2001 From: Richard Downer Date: Mon, 10 Apr 2017 10:20:07 +0100 Subject: [PATCH 5/5] ElasticSearch: ulimit/sysctl configuration updates Newest version requires increased ulimit/sysctl values. Apply values immediately so that a reboot is not required. --- .../nosql/elasticsearch/ElasticSearchNodeSshDriver.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNodeSshDriver.java b/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNodeSshDriver.java index eff68ca9b..439096617 100644 --- a/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNodeSshDriver.java +++ b/software/nosql/src/main/java/org/apache/brooklyn/entity/nosql/elasticsearch/ElasticSearchNodeSshDriver.java @@ -52,6 +52,7 @@ public void install() { List commands = ImmutableList.builder() .add(BashCommands.sudo(String.format("cp %s %s", tmpLimitsFile, "/etc/security/limits.d/99-elasticsearch.conf"))) .add(BashCommands.sudo(String.format("cp %s %s", tmpSysctlFile, "/etc/sysctl.d/99-elasticsearch.conf"))) + .add(BashCommands.sudo(String.format("sysctl --load %s", "/etc/sysctl.d/99-elasticsearch.conf"))) .addAll(BashCommands.commandsToDownloadUrlsAs(urls, saveAs)) .add(String.format("tar zxvf %s", saveAs)) .build(); @@ -92,9 +93,12 @@ public void launch() { appendConfigIfPresent(commandBuilder, "discovery.zen.ping.multicast.enabled", ElasticSearchNode.MULTICAST_ENABLED); appendConfigIfPresent(commandBuilder, "discovery.zen.ping.unicast.enabled", ElasticSearchNode.UNICAST_ENABLED); commandBuilder.append(" > out.log 2> err.log < /dev/null"); + final List cmds = ImmutableList.of( + String.format("for p in $( pidof bash ); do echo set prlimit on process $p; %s; done", BashCommands.sudo("prlimit --nofile=65536:65536 --pid=$p")), + commandBuilder.toString()); newScript(MutableMap.of("usePidFile", false), LAUNCHING) .updateTaskAndFailOnNonZeroResultCode() - .body.append(commandBuilder.toString()) + .body.append(cmds) .execute(); }