WHIRR-240. [HBase] Enable support for HBase 0.90.x (Bruno Dumon via a…

…savu)

git-svn-id: https://svn.apache.org/repos/asf/incubator/whirr/trunk@1153757 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -13,6 +13,8 @@ Trunk (unreleased changes)
 
     WHIRR-260. Support Spot Instances (asavu)
 
+    WHIRR-240. [HBase] Enable support for HBase 0.90.x (Bruno Dumon via asavu)
+
   IMPROVEMENTS
 
     WHIRR-28. Add examples module (asavu)

recipes/hbase-ec2-0.90.properties

@@ -0,0 +1,55 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# HBase 0.90.x on Cloudera Hadoop Cluster on AWS EC2
+# 
+
+# Read the Configuration Guide for more info:
+# http://incubator.apache.org/whirr/configuration-guide.html
+
+# Change the cluster name here
+whirr.cluster-name=hbase-0.90
+
+# Change the number of machines in the cluster here
+whirr.instance-templates=1 zookeeper+hadoop-namenode+hadoop-jobtracker+hbase-master,3 hadoop-datanode+hadoop-tasktracker+hbase-regionserver
+
+# For EC2 set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables.
+whirr.provider=aws-ec2
+whirr.identity=${env:AWS_ACCESS_KEY_ID}
+whirr.credential=${env:AWS_SECRET_ACCESS_KEY}
+
+# The size of the instance to use. See http://aws.amazon.com/ec2/instance-types/
+whirr.hardware-id=c1.xlarge
+# Ubuntu 10.04 LTS Lucid. See http://alestic.com/
+whirr.image-id=us-east-1/ami-da0cf8b3
+# If you choose a different location, make sure whirr.image-id is updated too
+whirr.location-id=us-east-1
+
+# By default use the user system SSH keys. Override them here.
+# whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
+# whirr.public-key-file=${whirr.private-key-file}.pub
+
+# The HBase version to use.
+whirr.hbase.tarball.url=http://apache.cu.be/hbase/hbase-0.90.3/hbase-0.90.3.tar.gz
+
+# The Hadoop version to use. See http://hbase.apache.org/book/hadoop.html
+# The default Hadoop version used by Whirr does not fulfill the HBase 0.90.x requirements.
+# Whirr will replace the hadoop-core jar in HBase with the one from the actually installed Hadoop.
+# This example uses Cloudera's CDH3.
+whirr.hadoop.tarball.url=http://archive.cloudera.com/cdh/3/hadoop-0.20.2-cdh3u1.tar.gz
+

services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHBaseServiceTest.java

@@ -48,7 +48,7 @@ public class CdhHBaseServiceTest {
   private static final byte[] VALUE = Bytes.toBytes("testValue");
 
   private static HBaseServiceController controller =
-    HBaseServiceController.getInstance();
+    HBaseServiceController.getInstance("whirr-hbase-test.properties");
 
   @BeforeClass
   public static void setUp() throws Exception {

services/hbase/src/main/resources/functions/configure_hbase.sh

@@ -130,6 +130,10 @@ function configure_hbase() {
  <name>hbase.client.retries.number</name>
  <value>100</value>
 </property>
+<property>
+ <name>hbase.zookeeper.recoverable.waittime</name>
+ <value>600000</value>
+</property>
 </configuration>
 EOF
 
@@ -169,6 +173,22 @@ EOF
 #mapred.servers=$MASTER_HOST:8649
 #EOF
 
+  # Replace Hadoop jar of HBase with the ones from the actually installed Hadoop version
+  # This assumes there will always be Hadoop installed on each HBase node
+
+  if [ -d /usr/local/hadoop ] ; then
+    # First, remove existing jar file
+    rm -f $HBASE_HOME/lib/hadoop*
+
+    # This makes the assumption there will be exactly one file matching
+    # The stars around core is because the file is named differently in CDH vs Apache
+    # Hadoop distributon (hadoop-core-version vs hadoop-version-core).
+    HADOOP_JAR=`ls /usr/local/hadoop-*/hadoop*core*.jar`
+    ln -s $HADOOP_JAR $HBASE_HOME/lib/hadoop-core.jar
+  else
+    echo Copy hadoop jar to HBase error: did not find your Hadoop installation
+  fi
+
   # keep PID files in a non-temporary directory
   sed -i -e "s|# export HBASE_PID_DIR=.*|export HBASE_PID_DIR=/var/run/hbase|" \
     $HBASE_CONF_DIR/hbase-env.sh

services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBase089ServiceTest.java

@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.hbase.integration;
+
+import org.junit.BeforeClass;
+
+public class HBase089ServiceTest extends HBaseServiceTest {
+  @BeforeClass
+  public static void setUp() throws Exception {
+    controller = HBaseServiceController.getInstance("whirr-hbase-test.properties");
+    controller.ensureClusterRunning();
+  }
+}

services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBase090ServiceTest.java

@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.hbase.integration;
+
+import org.junit.BeforeClass;
+
+public class HBase090ServiceTest extends HBaseServiceTest {
+  @BeforeClass
+  public static void setUp() throws Exception {
+    controller = HBaseServiceController.getInstance("whirr-hbase-0.90-test.properties");
+    controller.ensureClusterRunning();
+  }
+}

services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBaseServiceController.java

@@ -20,6 +20,8 @@
 
 import java.io.IOException;
 import java.net.InetAddress;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Map.Entry;
 
 import org.apache.commons.configuration.CompositeConfiguration;
@@ -47,14 +49,20 @@
 
 public class HBaseServiceController {
 
+  private final String configResource;
+
   private static final Logger LOG =
     LoggerFactory.getLogger(HBaseServiceController.class);
 
-  private static final HBaseServiceController INSTANCE =
-    new HBaseServiceController();
+  private static final Map<String, HBaseServiceController> INSTANCES = new HashMap<String, HBaseServiceController>();
 
-  public static HBaseServiceController getInstance() {
-    return INSTANCE;
+  public static HBaseServiceController getInstance(String configResource) {
+    HBaseServiceController controller = INSTANCES.get(configResource);
+    if (controller == null) {
+      controller = new HBaseServiceController(configResource);
+      INSTANCES.put(configResource, controller);
+    }
+    return controller;
   }
 
   private boolean running;
@@ -64,7 +72,8 @@ public static HBaseServiceController getInstance() {
   private Cluster cluster;
   private Hbase.Client thriftClient;
 
-  private HBaseServiceController() {
+  private HBaseServiceController(String configResource) {
+    this.configResource = configResource;
   }
 
   public synchronized boolean ensureClusterRunning() throws Exception {
@@ -83,7 +92,7 @@ public synchronized void startup() throws Exception {
     if (System.getProperty("config") != null) {
       config.addConfiguration(new PropertiesConfiguration(System.getProperty("config")));
     }
-    config.addConfiguration(new PropertiesConfiguration("whirr-hbase-test.properties"));
+    config.addConfiguration(new PropertiesConfiguration(this.configResource));
     clusterSpec = ClusterSpec.withTemporaryKeys(config);
     controller = new ClusterController();
 

services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBaseServiceTest.java

@@ -33,10 +33,9 @@
 import org.apache.hadoop.hbase.thrift.generated.TRowResult;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.AfterClass;
-import org.junit.BeforeClass;
 import org.junit.Test;
 
-public class HBaseServiceTest {
+public abstract class HBaseServiceTest {
 
   private static final byte[] FIRST = Bytes.toBytes("");
   private static final byte[] TABLE = Bytes.toBytes("testtable");
@@ -46,13 +45,7 @@ public class HBaseServiceTest {
   private static final byte[] COLUMN = Bytes.toBytes("testFamily1:testColumn");
   private static final byte[] VALUE = Bytes.toBytes("testValue");
 
-  private static HBaseServiceController controller =
-    HBaseServiceController.getInstance();
-
-  @BeforeClass
-  public static void setUp() throws Exception {
-    controller.ensureClusterRunning();
-  }
+  protected static HBaseServiceController controller;
 
   @AfterClass
   public static void tearDown() throws Exception {

services/hbase/src/test/resources/whirr-hbase-0.90-test.properties

@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+whirr.cluster-name=hbaseclustertest
+whirr.instance-templates=1 hadoop-namenode+hadoop-jobtracker+zookeeper+hbase-master+hbase-thriftserver,1 hadoop-datanode+hadoop-tasktracker+hbase-regionserver
+
+whirr.provider=${sys:whirr.test.provider}
+whirr.identity=${sys:whirr.test.identity}
+whirr.credential=${sys:whirr.test.credential}
+
+whirr.hbase.tarball.url=http://apache.cu.be/hbase/hbase-0.90.3/hbase-0.90.3.tar.gz
+whirr.hadoop.tarball.url=http://archive.cloudera.com/cdh/3/hadoop-0.20.2-cdh3u0.tar.gz

services/hbase/src/test/resources/whirr-hbase-test.properties

@@ -16,7 +16,7 @@
 # limitations under the License.
 #
 whirr.cluster-name=hbaseclustertest
-whirr.instance-templates=1 hadoop-namenode+hadoop-jobtracker+hbase-master,1 zookeeper+hbase-thriftserver,1 hadoop-datanode+hadoop-tasktracker+hbase-regionserver
+whirr.instance-templates=1 hadoop-namenode+hadoop-jobtracker+zookeeper+hbase-master+hbase-thriftserver,1 hadoop-datanode+hadoop-tasktracker+hbase-regionserver
 
 whirr.provider=${sys:whirr.test.provider}
 whirr.identity=${sys:whirr.test.identity}