WHIRR-297. Separate ZooKeeper and ElasticSearch install and configura…

…tion scripts into more generic functions (asavu)

git-svn-id: https://svn.apache.org/repos/asf/incubator/whirr/trunk@1099716 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -56,6 +56,9 @@ Trunk (unreleased changes)
 
     WHIRR-291. Add "noop" role useful just for provisioning (asavu)
 
+    WHIRR-297. Separate ZooKeeper and ElasticSearch install and configuration 
+    scripts into more generic functions (asavu)
+
   BUG FIXES
 
     WHIRR-253. ZooKeeper service should only authorize ingress to ZooKeeper 

core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java

@@ -60,12 +60,15 @@ public Cluster execute(ClusterSpec clusterSpec, Cluster cluster) throws IOExcept
     Cluster newCluster = cluster;
     for (InstanceTemplate instanceTemplate : clusterSpec.getInstanceTemplates()) {
       StatementBuilder statementBuilder = new StatementBuilder();
+
       ComputeServiceContext computServiceContext = // TODO: shouldn't create lots of these
         ComputeServiceContextBuilder.build(getComputeServiceContextFactory(), clusterSpec);
       FirewallManager firewallManager = new FirewallManager(computServiceContext,
           clusterSpec, newCluster);
+
       ClusterActionEvent event = new ClusterActionEvent(getAction(),
           clusterSpec, newCluster, statementBuilder, firewallManager);
+
       eventMap.put(instanceTemplate, event);
       for (String role : instanceTemplate.getRoles()) {
         ClusterActionHandler handler = handlerMap.get(role);

core/src/main/resources/functions/install_service.sh

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+function install_service() {
+    if [ "$1" != "" ]; then
+        if which update-rc.d >/dev/null 2>&1; then
+            update-rc.d $1 defaults
+        fi
+        if which chkconfig >/dev/null 2>&1; then
+            chkconfig --add $1
+        fi
+    fi
+}

core/src/main/resources/functions/install_tarball.sh

@@ -16,13 +16,16 @@
 #
 function install_tarball() {
   if [[ "$1" != "" ]]; then
-    # Download a .tar.gz file and extract to /usr/local
+    # Download a .tar.gz file and extract to target dir
 
     local tar_url=$1
     local tar_file=`basename $tar_url`
     local tar_file_md5=`basename $tar_url.md5`
 
-    local curl="curl --silent --show-error --fail --connect-timeout 10 --max-time 600"
+    local target=${2:-/usr/local/}
+    mkdir -p $target
+
+    local curl="curl -L --silent --show-error --fail --connect-timeout 10 --max-time 600 --retry 5"
     # any download should take less than 10 minutes
 
     for retry_count in `seq 1 3`;
@@ -54,7 +57,7 @@ function install_tarball() {
       exit 1
     fi
 
-    tar xzf $tar_file -C /usr/local
+    tar xzf $tar_file -C $target
     rm -f $tar_file $tar_file_md5
   fi
 }

core/src/main/resources/functions/remove_service.sh

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+function remove_service() {
+    if [ "$1" != "" ]; then
+        if which update-rc.d >/dev/null 2>&1; then
+            update-rc.d -f $1 remove
+        fi
+        if which chkconfig >/dev/null 2>&1; then
+            chkconfig --del $1
+        fi
+    fi
+}

services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchHandler.java

@@ -52,6 +52,7 @@ protected void beforeBootstrap(ClusterActionEvent event) throws IOException {
     Configuration config = spec.getConfiguration();
 
     addStatement(event, call("install_java"));
+    addStatement(event, call("install_tarball"));
 
     String tarurl = prepareRemoteFileUrl(event,
         config.getString("whirr.elasticsearch.tarball.url", ""));
@@ -74,5 +75,6 @@ protected void beforeConfigure(ClusterActionEvent event) throws IOException {
       ElasticSearchConfigurationBuilder.build("/tmp/elasticsearch.yml", config));
     addStatement(event, call("configure_elasticsearch",
       config.getStringArray("es.plugins")));
+    addStatement(event, call("start_elasticsearch"));
   }
 }

services/elasticsearch/src/main/resources/functions/cleanup_elasticsearch.sh

@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+function cleanup_elasticsearch() {
+    cd /usr/local/elasticsearch-*
+
+    ./bin/service/elasticsearch stop
+    ./bin/service/elasticsearch remove
+
+    rm -rf /usr/local/elasticsearch-*
+}

services/elasticsearch/src/main/resources/functions/configure_elasticsearch.sh

@@ -22,11 +22,9 @@ function configure_elasticsearch() {
         ./bin/plugin install $plugin
     done
 
-    cp /tmp/elasticsearch.yml config/elasticsearch.yml
-
-    # use 80% of memory for jvm heap
-    local MAXMEM=$(($(free|awk '/^Mem:/{print $2}') * 8 / 10 / 1024))m
+    # TODO allow user to set the amount of memory to use
+    # local MAXMEM=$(($(free|awk '/^Mem:/{print $2}') * 8 / 10 / 1024))m
 
-    ./bin/elasticsearch -Xmx$MAXMEM -Xms$MAXMEM -p elasticsearch.pid
+    cp /tmp/elasticsearch.yml config/elasticsearch.yml
 }
 

services/elasticsearch/src/main/resources/functions/install_elasticsearch.sh

@@ -15,28 +15,21 @@
 # limitations under the License.
 #
 function install_elasticsearch() {
-    local CURL="curl -L --silent --show-error --fail --connect-timeout 10 --max-time 600 --retry 5"
+
+    # TODO Run ElasticSearch as non-root-user
+    # http://www.elasticsearch.org/tutorials/2011/02/22/running-elasticsearch-as-a-non-root-user.html
 
     local ES_URL=${1:-http://github.com/downloads/elasticsearch/elasticsearch/elasticsearch-0.15.2.tar.gz}
-    local TAR_FILE=`basename $ES_URL`
+    install_tarball $ES_URL
 
-    for retry_count in `seq 1 3`
-    do
-      $CURL -O $ES_URL || true
-      if [ -f $TAR_FILE ]; then
-        break;
-      fi
-      if [ ! $retry_count -eq "3" ]; then
-        sleep 10
-      fi
-    done
+    # install the latest service wrapper for elasticsearch
+    local ES_WRAPPER=https://github.com/elasticsearch/elasticsearch-servicewrapper/tarball/master
+    install_tarball $ES_WRAPPER /tmp/
 
-    if [ -f $TAR_FILE ]; then
-      tar xzf $TAR_FILE -C /usr/local
-      rm -f $TAR_FILE
-    else
-      echo "Unable to download tar file from $ES_URL" >&2
-      exit 1
-    fi
-}
+    # move the service wrapper in place
+    mv /tmp/elasticsearch-elasticsearch-servicewrapper-*/service /usr/local/elasticsearch-*/bin/
+    cd /usr/local/elasticsearch-*
 
+    # ensure that elasticsearch will start after reboot
+    ./bin/service/elasticsearch install
+}

services/elasticsearch/src/main/resources/functions/start_elasticsearch.sh

@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+function start_elasticsearch() {
+    cd /usr/local/elasticsearch-*
+    ./bin/service/elasticsearch start
+}

services/elasticsearch/src/main/resources/functions/stop_elasticsearch.sh

@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+function stop_elasticsearch() {
+    cd /usr/local/elasticsearch-*
+    ./bin/service/elasticsearch stop
+}

services/zookeeper/src/main/java/org/apache/whirr/service/zookeeper/ZooKeeperClusterActionHandler.java

@@ -63,6 +63,10 @@ protected void beforeBootstrap(ClusterActionEvent event) throws IOException {
     addStatement(event, call("install_java"));
     addStatement(event, call("install_tarball"));
 
+    /* register utility functions for managing init scripts */
+    addStatement(event, call("install_service"));
+    addStatement(event, call("remove_service"));
+
     String tarurl = config.getString("whirr.zookeeper.tarball.url");
     addStatement(event, call("install_zookeeper",
       prepareRemoteFileUrl(event, tarurl)));
@@ -82,12 +86,14 @@ protected void beforeConfigure(ClusterActionEvent event) throws IOException, Int
     String servers = Joiner.on(' ').join(getPrivateIps(ensemble));
     addStatement(event, call("configure_zookeeper", "-c",
         clusterSpec.getProvider(), servers));
+    addStatement(event, call("start_zookeeper"));
   }
 
   @Override
   protected void afterConfigure(ClusterActionEvent event) {
     ClusterSpec clusterSpec = event.getClusterSpec();
     Cluster cluster = event.getCluster();
+
     LOG.info("Completed configuration of {}", clusterSpec.getClusterName());
     String hosts = Joiner.on(',').join(getHosts(cluster.getInstancesMatching(
       role(ZOOKEEPER_ROLE))));

services/zookeeper/src/main/resources/functions/cleanup_zookeeper.sh

@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+function cleanup_zookeeper() {
+    /etc/init.d/zookeeper stop
+    remove_service zookeeper
+    # TODO remove cron job
+    rm -rf /usr/local/zookeeper* /etc/zookeeper /var/log/zookeeper /mnt/zookeeper
+    rm -f /etc/init.d/zookeeper
+}

services/zookeeper/src/main/resources/functions/configure_zookeeper.sh

@@ -77,8 +77,4 @@ EOF
     fi
     echo $myid > $myid_file
   fi
-
-  # Now that it's configured, start ZooKeeper
-  nohup /etc/rc.local &
-
 }

services/zookeeper/src/main/resources/functions/install_zookeeper.sh

@@ -44,11 +44,11 @@ function install_zookeeper() {
 
   CRON="0 0 * * * java -cp $ZK_JAR:$ZK_LOG4J_JAR:$ZK_CONF_DIR org.apache.zookeeper.server.PurgeTxnLog $ZK_DATA_DIR $ZK_DATA_DIR -n 10"
   crontab -l 2>/dev/null | { cat; echo "$CRON"; } | sort | uniq | crontab -
-
-  # Ensure ZooKeeper starts on boot
-  sed -i -e "s/exit 0//" /etc/rc.local
-cat >> /etc/rc.local <<EOF
-ZOOCFGDIR=$ZK_CONF_DIR $ZOOKEEPER_HOME/bin/zkServer.sh start > /dev/null 2>&1 &
-EOF
 
+  cat >/etc/init.d/zookeeper <<EOF
+#!/bin/bash
+ZOOCFGDIR=$ZK_CONF_DIR $ZOOKEEPER_HOME/bin/zkServer.sh \$@ > /dev/null 2>&1 &
+EOF
+  chmod +x /etc/init.d/zookeeper
+  install_service zookeeper
 }

services/zookeeper/src/main/resources/functions/start_zookeeper.sh

@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+function start_zookeeper() {
+    /etc/init.d/zookeeper start
+}

services/zookeeper/src/main/resources/functions/stop_zookeeper.sh

@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+function stop_zookeeper() {
+    /etc/init.d/zookeeper stop
+}