Permalink
Browse files

WHIRR-186. [HBase] Add version support configurable in properties file.

git-svn-id: https://svn.apache.org/repos/asf/incubator/whirr/trunk@1064216 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information...
1 parent cd10919 commit e1632b67058149e297f075f66a44db9e23d09699 @larsgeorge larsgeorge committed Jan 27, 2011
View
@@ -16,6 +16,8 @@ Trunk (unreleased changes)
WHIRR-170. Instances should be started in the order specified in
the template (tomwhite via asavu)
+ WHIRR-186. [HBase] Add version support configurable in properties file.
+
Release 0.3.0 - 2011-01-15
NEW FEATURES
@@ -17,26 +17,44 @@
#
# Install Apache HBase.
#
+# Call with the following arguments
+# -c <cloud provider>
+# -u <hbase tarball url>
set -x
set -e
-################################################################################
-# Initialize variables
-################################################################################
-
+# get parameters
CLOUD_PROVIDER=
-while getopts "c:" OPTION; do
+HBASE_TAR_URL=
+while getopts "c:u:" OPTION; do
case $OPTION in
c)
CLOUD_PROVIDER="$OPTARG"
;;
+ u)
+ HBASE_TAR_URL="$OPTARG"
+ ;;
esac
done
-HBASE_VERSION=${HBASE_VERSION:-0.89.20100924}
-HBASE_HOME=/usr/local/hbase-$HBASE_VERSION
+# assign default URL if no other given (optional)
+HBASE_TAR_URL=${HBASE_TAR_URL:-http://archive.apache.org/dist/hbase/hbase-0.90.0/hbase-0.90.0.tar.gz}
+# derive details from the URL
+HBASE_TAR_FILE=${HBASE_TAR_URL##*/}
+HBASE_TAR_MD5_FILE=$HBASE_TAR_FILE.md5
+# extract "version" or the name of the directory contained in the tarball,
+# but since hbase has used different namings use the directory instead.
+HBASE_VERSION=${HBASE_TAR_URL%/*.tar.gz}
+HBASE_VERSION=${HBASE_VERSION##*/}
+# simple check that we have a proper URL or default to use filename
+if [[ "${HBASE_VERSION:0:5}" != "hbase" ]]; then
+ HBASE_VERSION=${HBASE_TAR_FILE%.tar.gz}
+fi
+HBASE_HOME=/usr/local/$HBASE_VERSION
+HBASE_CONF_DIR=$HBASE_HOME/conf
+# update the debian or redhat repos
function update_repo() {
if which dpkg &> /dev/null; then
sudo apt-get update
@@ -45,7 +63,7 @@ function update_repo() {
fi
}
-# Install a list of packages on debian or redhat as appropriate
+# install a list of packages on debian or redhat
function install_packages() {
if which dpkg &> /dev/null; then
apt-get update
@@ -57,6 +75,7 @@ function install_packages() {
fi
}
+# install hbase
function install_hbase() {
if ! id hadoop &> /dev/null; then
useradd hadoop
@@ -70,10 +89,10 @@ function install_hbase() {
ulimit -n 65535
# up epoll limits; ok if this fails, only valid for kernels 2.6.27+
set +e
- sysctl -w fs.epoll.max_user_instances=65535 > /dev/null 2>&1
+ sysctl -w fs.epoll.max_user_instances=4096 > /dev/null 2>&1
set -e
# if there is no hosts file then provide a minimal one
- [ ! -f /etc/hosts ] && echo "127.0.0.1 localhost" > /etc/hosts
+ [ ! -f /etc/hosts ] && echo "127.0.0.1 localhost" > /etc/hosts
# Reformat sdb as xfs
#umount /mnt
@@ -99,29 +118,25 @@ function install_hbase() {
#done
# install HBase tarball
- hbase_tar_url=http://archive.apache.org/dist/hbase/hbase-$HBASE_VERSION/hbase-${HBASE_VERSION}-bin.tar.gz
- hbase_tar_file=`basename $hbase_tar_url`
- hbase_tar_md5_file=`basename $hbase_tar_url.md5`
-
curl="curl --retry 3 --silent --show-error --fail"
for i in `seq 1 3`;
do
- $curl -O $hbase_tar_url
- $curl -O $hbase_tar_url.md5
- if md5sum -c $hbase_tar_md5_file; then
+ $curl -O $HBASE_TAR_URL
+ $curl -O $HBASE_TAR_URL.md5
+ if md5sum -c $HBASE_TAR_MD5_FILE; then
break;
else
- rm -f $hbase_tar_file $hbase_tar_md5_file
+ rm -f $HBASE_TAR_FILE $HBASE_TAR_MD5_FILE
fi
done
- if [ ! -e $hbase_tar_file ]; then
- echo "Failed to download $hbase_tar_url. Aborting."
+ if [ ! -e $HBASE_TAR_FILE ]; then
+ echo "Failed to download $HBASE_TAR_URL. Aborting."
exit 1
fi
- tar zxf $hbase_tar_file -C /usr/local
- rm -f $hbase_tar_file $hbase_tar_md5_file
+ tar zxf $HBASE_TAR_FILE -C /usr/local
+ rm -f $HBASE_TAR_FILE $HBASE_TAR_MD5_FILE
echo "export HBASE_HOME=$HBASE_HOME" >> ~root/.bashrc
echo 'export PATH=$JAVA_HOME/bin:$HBASE_HOME/bin:$PATH' >> ~root/.bashrc
@@ -22,22 +22,21 @@
# -q <zookeeper quorum>
# -p <port>
# -c <cloud provider>
+# -u <hbase tarball url>
set -x
set -e
-################################################################################
-# Initialize variables
-################################################################################
-
ROLES=$1
shift
+# get parameters
MASTER_HOST=
ZOOKEEKER_QUORUM=
-CLOUD_PROVIDER=
PORT=
-while getopts "m:q:p:c:" OPTION; do
+CLOUD_PROVIDER=
+HBASE_TAR_URL=
+while getopts "m:q:p:c:u:" OPTION; do
case $OPTION in
m)
MASTER_HOST="$OPTARG"
@@ -51,9 +50,13 @@ while getopts "m:q:p:c:" OPTION; do
c)
CLOUD_PROVIDER="$OPTARG"
;;
+ u)
+ HBASE_TAR_URL="$OPTARG"
+ ;;
esac
done
+# determine machine name
case $CLOUD_PROVIDER in
ec2)
# Use public hostname for EC2
@@ -64,11 +67,22 @@ case $CLOUD_PROVIDER in
;;
esac
-HBASE_VERSION=${HBASE_VERSION:-0.89.20100924}
-HBASE_HOME=/usr/local/hbase-$HBASE_VERSION
+# assign default URL if no other given (optional)
+HBASE_TAR_URL=${HBASE_TAR_URL:-http://archive.apache.org/dist/hbase/hbase-0.90.0/hbase-0.90.0.tar.gz}
+# derive details from the URL
+HBASE_TAR_FILE=${HBASE_TAR_URL##*/}
+# extract "version" or the name of the directory contained in the tarball,
+# but since hbase has used different namings use the directory instead.
+HBASE_VERSION=${HBASE_TAR_URL%/*.tar.gz}
+HBASE_VERSION=${HBASE_VERSION##*/}
+# simple check that we have a proper URL or default to use filename
+if [[ "${HBASE_VERSION:0:5}" != "hbase" ]]; then
+ HBASE_VERSION=${HBASE_TAR_FILE%.tar.gz}
+fi
+HBASE_HOME=/usr/local/$HBASE_VERSION
HBASE_CONF_DIR=$HBASE_HOME/conf
-# Configure HBase by setting up disks and site file
+# configure hbase by setting up disks and site file
function configure_hbase() {
case $CLOUD_PROVIDER in
ec2)
@@ -91,7 +105,7 @@ function configure_hbase() {
##############################################################################
# Modify this section to customize your HBase cluster.
##############################################################################
-cat > $HBASE_HOME/conf/hbase-site.xml <<EOF
+ cat > $HBASE_CONF_DIR/hbase-site.xml <<EOF
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
@@ -125,13 +139,15 @@ cat > $HBASE_HOME/conf/hbase-site.xml <<EOF
</property>
</configuration>
EOF
-# Override JVM options
-cat >> $HBASE_HOME/conf/hbase-env.sh <<EOF
+
+ # override JVM options
+ cat >> $HBASE_CONF_DIR/hbase-env.sh <<EOF
export HBASE_MASTER_OPTS="-Xms1000m -Xmx1000m -Xmn256m -XX:+UseConcMarkSweepGC -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/mnt/hbase/logs/hbase-master-gc.log"
export HBASE_REGIONSERVER_OPTS="-Xms2000m -Xmx2000m -Xmn256m -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88 -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/mnt/hbase/logs/hbase-regionserver-gc.log"
EOF
-# Configure HBase for Ganglia
-cat > $HBASE_HOME/conf/hadoop-metrics.properties <<EOF
+
+ # configure hbase for ganglia
+ cat > $HBASE_CONF_DIR/hadoop-metrics.properties <<EOF
dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext
dfs.period=10
dfs.servers=$MASTER_HOST:8649
@@ -143,11 +159,11 @@ jvm.period=10
jvm.servers=$MASTER_HOST:8649
EOF
- # Update classpath to include HBase jars and config
+ # update classpath to include hbase jars and config
# cat >> $HADOOP_HOME/conf/hadoop-env.sh <<EOF
-#HADOOP_CLASSPATH="$HBASE_HOME/hbase-${HBASE_VERSION}.jar:$HBASE_HOME/lib/zookeeper-3.3.1.jar:$HBASE_HOME/conf"
+#HADOOP_CLASSPATH="$HBASE_HOME/${HBASE_VERSION}.jar:$HBASE_HOME/lib/zookeeper-3.3.1.jar:$HBASE_CONF_DIR"
#EOF
- # Configure Hadoop for Ganglia
+ # configure Hadoop for Ganglia
# cat > $HADOOP_HOME/conf/hadoop-metrics.properties <<EOF
#dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext
#dfs.period=10
@@ -160,21 +176,21 @@ EOF
#mapred.servers=$MASTER_HOST:8649
#EOF
- # Keep PID files in a non-temporary directory
+ # keep PID files in a non-temporary directory
sed -i -e "s|# export HBASE_PID_DIR=.*|export HBASE_PID_DIR=/var/run/hbase|" \
$HBASE_CONF_DIR/hbase-env.sh
mkdir -p /var/run/hbase
chown -R hadoop:hadoop /var/run/hbase
- # Set SSH options within the cluster
+ # set SSH options within the cluster
sed -i -e 's|# export HBASE_SSH_OPTS=.*|export HBASE_SSH_OPTS="-o StrictHostKeyChecking=no"|' \
$HBASE_CONF_DIR/hbase-env.sh
- # Disable IPv6
+ # disable IPv6
sed -i -e 's|# export HBASE_OPTS=.*|export HBASE_OPTS="-Djava.net.preferIPv4Stack=true"|' \
$HBASE_CONF_DIR/hbase-env.sh
- # HBase logs should be on the /mnt partition
+ # hbase logs should be on the /mnt partition
sed -i -e 's|# export HBASE_LOG_DIR=.*|export HBASE_LOG_DIR=/var/log/hbase/logs|' \
$HBASE_CONF_DIR/hbase-env.sh
rm -rf /var/log/hbase
@@ -184,6 +200,7 @@ EOF
chown -R hadoop:hadoop /var/log/hbase
}
+# helper to start daemons, used below
function start_daemon() {
if which dpkg &> /dev/null; then
AS_HADOOP="su -s /bin/bash - hadoop -c"
@@ -26,7 +26,6 @@
import org.apache.whirr.net.DnsUtil;
import org.apache.whirr.service.Cluster;
import org.apache.whirr.service.ClusterActionEvent;
-import org.apache.whirr.service.ClusterActionHandlerSupport;
import org.apache.whirr.service.ClusterSpec;
import org.apache.whirr.service.ComputeServiceContextBuilder;
import org.apache.whirr.service.jclouds.FirewallSettings;
@@ -36,7 +35,7 @@
/**
* Provides a base class for servers like REST or Avro.
*/
-public class BasicServerClusterActionHandler extends ClusterActionHandlerSupport {
+public class BasicServerClusterActionHandler extends HBaseClusterActionHandler {
private final String role;
private final int defaultPort;
@@ -59,10 +58,13 @@ protected void beforeBootstrap(ClusterActionEvent event) throws IOException {
addRunUrl(event, "util/configure-hostnames",
HBaseConstants.PARAM_PROVIDER, clusterSpec.getProvider());
addRunUrl(event, "sun/java/install");
- String hbaseInstallRunUrl = clusterSpec.getConfiguration().getString(
+ String hbaseInstallRunUrl = getConfiguration(clusterSpec).getString(
HBaseConstants.KEY_INSTALL_RUNURL, HBaseConstants.SCRIPT_INSTALL);
- addRunUrl(event, hbaseInstallRunUrl,
- HBaseConstants.PARAM_PROVIDER, clusterSpec.getProvider());
+ String tarurl = getConfiguration(clusterSpec).getString(
+ HBaseConstants.KEY_TARBALL_URL);
+ addRunUrl(event, hbaseInstallRunUrl,
+ HBaseConstants.PARAM_PROVIDER, clusterSpec.getProvider(),
+ HBaseConstants.PARAM_TARBALL_URL, tarurl);
event.setTemplateBuilderStrategy(new HBaseTemplateBuilderStrategy());
}
@@ -73,7 +75,7 @@ protected void beforeConfigure(ClusterActionEvent event)
Cluster cluster = event.getCluster();
int port = defaultPort;
if (configKeyPort != null) {
- port = clusterSpec.getConfiguration().getInt(configKeyPort, defaultPort);
+ port = getConfiguration(clusterSpec).getInt(configKeyPort, defaultPort);
}
Cluster.Instance instance = cluster.getInstanceMatching(
@@ -85,15 +87,19 @@ protected void beforeConfigure(ClusterActionEvent event)
FirewallSettings.authorizeIngress(computeServiceContext, instance,
clusterSpec, port);
- String hbaseConfigureRunUrl = clusterSpec.getConfiguration().getString(
+ String hbaseConfigureRunUrl = getConfiguration(clusterSpec).getString(
HBaseConstants.KEY_CONFIGURE_RUNURL,
HBaseConstants.SCRIPT_POST_CONFIGURE);
String master = DnsUtil.resolveAddress(masterPublicAddress.getHostAddress());
String quorum = ZooKeeperCluster.getHosts(cluster);
- addRunUrl(event, hbaseConfigureRunUrl, role,
- HBaseConstants.PARAM_MASTER, master,
- HBaseConstants.PARAM_QUORUM, quorum,
- HBaseConstants.PARAM_PORT, Integer.toString(port),
- HBaseConstants.PARAM_PROVIDER, clusterSpec.getProvider());
+ String tarurl = getConfiguration(clusterSpec).getString(
+ HBaseConstants.KEY_TARBALL_URL);
+ addRunUrl(event, hbaseConfigureRunUrl, role,
+ HBaseConstants.PARAM_MASTER, master,
+ HBaseConstants.PARAM_QUORUM, quorum,
+ HBaseConstants.PARAM_PORT, Integer.toString(port),
+ HBaseConstants.PARAM_PROVIDER, clusterSpec.getProvider(),
+ HBaseConstants.PARAM_TARBALL_URL, tarurl);
}
+
}
@@ -26,7 +26,7 @@
public class HBaseCluster {
public static InetAddress getMasterPublicAddress(Cluster cluster) {
return cluster.getInstanceMatching(
- RolePredicates.role(HBaseMasterClusterActionHandler.ROLE))
+ RolePredicates.role(HBaseMasterClusterActionHandler.ROLE))
.getPublicAddress();
}
}
Oops, something went wrong.

0 comments on commit e1632b6

Please sign in to comment.