magpie-setup-hbase

#!/bin/bash
#############################################################################
#  Copyright (C) 2013 Lawrence Livermore National Security, LLC.
#  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
#  Written by Albert Chu <chu11@llnl.gov>
#  LLNL-CODE-644248
#  
#  This file is part of Magpie, scripts for running Hadoop on
#  traditional HPC systems.  For details, see <URL>.
#  
#  Magpie is free software; you can redistribute it and/or modify it
#  under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#  
#  Magpie is distributed in the hope that it will be useful, but
#  WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  General Public License for more details.
#  
#  You should have received a copy of the GNU General Public License
#  along with Magpie.  If not, see <http://www.gnu.org/licenses/>.
#############################################################################

# This script sets up configuration files for jobs.  For the most
# part, it shouldn't be editted.  See job submission files for
# configuration details.

if [ "${HBASE_SETUP}" != "yes" ]
then
    exit 0
fi

source ${MAGPIE_SCRIPTS_HOME}/magpie-submission-convert
source ${MAGPIE_SCRIPTS_HOME}/magpie-common-exports
source ${MAGPIE_SCRIPTS_HOME}/magpie-common-functions

# hbasenoderank set if succeed
if ! Magpie_am_I_a_hbase_node
then
    exit 0
fi

if [ "${HBASE_MASTER_DAEMON_HEAP_MAX}X" != "X" ]
then
    hbasemasterdaemonheapmax="${HBASE_MASTER_DAEMON_HEAP_MAX}"
else
    hbasemasterdaemonheapmax="1000"
fi 

if [ "${HBASE_REGIONSERVER_DAEMON_HEAP_MAX}X" != "X" ]
then
    hbaseregionserverdaemonheapmax="${HBASE_REGIONSERVER_DAEMON_HEAP_MAX}"
else
    hbaseregionserverdaemonheapmax="16000"

    memtotal=`cat /proc/meminfo | grep MemTotal | awk '{print $2}'`
    memtotalmeg=$(echo "(${memtotal} / 1024)" | bc -l | xargs printf "%1.0f")
    # if memtotalmeg * 50% is less than hbaseregionserverdaemonheapmax, drop it down
    hbaseresourcememory=$(echo "${memtotalmeg} * .5" | bc -l | xargs printf "%1.0f")

    if [ "${hbaseresourcememory}" -lt "${hbaseregionserverdaemonheapmax}" ]
    then
	hbaseregionserverdaemonheapmax=${hbaseresourcememory}
    fi
fi

# My experience default 30 is low.  We'll say 50% of nodes, floor 60, ceiling 200
hbaseregionserverhandlercount=$(echo "${HBASE_REGIONSERVER_COUNT} * .5" | bc -l | xargs printf "%1.0f")
if [ "${hbaseregionserverhandlercount}" -lt "60" ]
then
    hbaseregionserverhandlercount=60
fi

if [ "${hbaseregionserverhandlercount}" -gt "200" ]
then
    hbaseregionserverhandlercount=200
fi

#
# Setup file system to use
#

# XXX how deal w/ fedreation?

# sets hadooptmpdir and fsdefault
Magpie_calculate_hadoop_filesystem_paths ${hbasenoderank}

hbasetmpdir="${hadooptmpdir}/hbase"
hbasetmpdirsubst=`echo "${hbasetmpdir}" | sed "s/\\//\\\\\\\\\//g"`
hbaserootdir="${fsdefault}/hbase"
hbaserootdirsubst=`echo ${hbaserootdir} | sed "s/\\//\\\\\\\\\//g"`

if [ ! -d "${hbasetmpdir}" ]
then
    mkdir -p ${hbasetmpdir}
    if [ $? -ne 0 ] ; then
	echo "mkdir failed making ${hbasetmpdir}"
	exit 1
    fi
fi

zookeepernodes=`cat ${ZOOKEEPER_CONF_DIR}/zookeeper_slaves`

hbasezookeeperquorum=""
for zookeepernode in ${zookeepernodes}
do
    if [ "${hbasezookeeperquorum}X" == "X" ]
    then
	hbasezookeeperquorum=${zookeepernode}
    else
	hbasezookeeperquorum="${hbasezookeeperquorum},${zookeepernode}"
    fi
done

# Sets hbase_slave_timeout
Magpie_calculate_stop_timeouts

openfiles=`ulimit -n`
if [ "${openfiles}" != "unlimited" ]
then
    openfileshardlimit=`ulimit -H -n`

    # we estimate 16384 per 64 nodes, minimum 32768, max 131072.
    # Obviously depends on many factors, but it's a reasonble and safe
    # over-estimate calculated based on experience.
    openfilesslavecount=`expr ${HBASE_REGIONSERVER_COUNT} \/ 64`
    openfilescount=`expr ${openfilesslavecount} \* 16384`
    if [ "${openfilescount}" -lt "32768" ]
    then
	openfilescount=32768
    fi
    if [ "${openfilescount}" -gt "131072" ]
    then
	openfilescount=131072
    fi
    
    if [ "${openfileshardlimit}" != "unlimited" ]
    then
        if [ ${openfilescount} -gt ${openfileshardlimit} ]
        then
            openfilescount=${openfileshardlimit}
        fi
    fi
else
    openfilescount="unlimited"
fi

userprocesses=`ulimit -u`
if [ "${userprocesses}" != "unlimited" ]
then
    userprocesseshardlimit=`ulimit -H -u`

    # we estimate 2048 per 100 nodes, minimum 4096, max 32768.
    userprocessesslavecount=`expr ${HBASE_REGIONSERVER_COUNT} \/ 100`
    userprocessescount=`expr ${userprocessesslavecount} \* 2048`
    if [ "${userprocessescount}" -lt "4096" ]
    then
	userprocessescount=4096
    fi
    if [ "${userprocessescount}" -gt "32768" ]
    then
	userprocessescount=32768
    fi
    
    if [ "${userprocesseshardlimit}" != "unlimited" ]
    then
        if [ ${userprocessescount} -gt ${userprocesseshardlimit} ]
        then
            userprocessescount=${userprocesseshardlimit}
        fi
    fi
else
    userprocessescount="unlimited"
fi

#
# Get config files for setup
#

if [ "${HBASE_CONF_FILES}X" == "X" ]
then
    hbaseconffiledir=${MAGPIE_SCRIPTS_HOME}/conf
else
    hbaseconffiledir=${HBASE_CONF_FILES}
fi

pre_hbasesitexml=${hbaseconffiledir}/hbase-site.xml
pre_hbaseenvsh=${hbaseconffiledir}/hbase-env.sh
pre_log4jproperties=${hbaseconffiledir}/hbase.log4j.properties

post_hbasesitexml=${HBASE_CONF_DIR}/hbase-site.xml
post_hbaseenvsh=${HBASE_CONF_DIR}/hbase-env.sh
post_log4jproperties=${HBASE_CONF_DIR}/log4j.properties

#
# Setup Hbase configuration files and environment files
#

cp ${pre_hbasesitexml} ${post_hbasesitexml}

HBASE_THIS_HOST=`hostname`

sed -i \
    -e "s/HBASETMPDIR/${hbasetmpdirsubst}/g" \
    -e "s/HBASEROOTDIR/${hbaserootdirsubst}/g" \
    -e "s/HBASEZOOKEEPERQUORUM/${hbasezookeeperquorum}/g" \
    -e "s/HBASEREGIONSERVERHANDLERCOUNT/${hbaseregionserverhandlercount}/g" \
    -e "s/HBASEMASTERPORT/${HBASE_MASTER_PORT}/g" \
    -e "s/HBASEMASTERINFOPORT/${HBASE_MASTER_INFO_PORT}/g" \
    -e "s/HBASEREGIONSERVERPORT/${HBASE_REGIONSERVER_PORT}/g" \
    -e "s/HBASEREGIONSERVERINFOPORT/${HBASE_REGIONSERVER_INFO_PORT}/g" \
    -e "s/HBASEZOOKEEPERPROPERTYCLIENTPORT/${ZOOKEEPER_CLIENT_PORT}/g" \
    -e "s/HBASEMASTERHOST/${HBASE_MASTER_NODE}/g" \
    -e "s/HBASE_HOST/${HBASE_THIS_HOST}/g" \
    ${post_hbasesitexml}

javahomesubst=`echo "${JAVA_HOME}" | sed "s/\\//\\\\\\\\\//g"`

cp ${pre_hbaseenvsh} ${post_hbaseenvsh}

sed -i \
    -e "s/HBASE_JAVA_HOME/${javahomesubst}/g" \
    -e "s/HBASEMASTERDAEMONHEAP/${hbasemasterdaemonheapmax}/g" \
    -e "s/HBASEREGIONSERVERDAEMONHEAP/${hbaseregionserverdaemonheapmax}/g" \
    ${post_hbaseenvsh}

echo "export HBASE_SLAVE_TIMEOUT=\"${hbase_slave_timeout}\"" >> ${post_hbaseenvsh}

echo "export HBASE_LOG_DIR=\"${HBASE_LOG_DIR}\"" >> ${post_hbaseenvsh}

if [ "${MAGPIE_REMOTE_CMD:-ssh}" != "ssh" ]
then
    echo "export HBASE_SSH_CMD=\"${MAGPIE_REMOTE_CMD}\"" >> ${post_hbaseenvsh}
fi
if [ "${MAGPIE_REMOTE_CMD_OPTS}X" != "X" ]
then
    echo "export HBASE_SSH_OPTS=\"${MAGPIE_REMOTE_CMD_OPTS}\"" >> ${post_hbaseenvsh}
fi

if [ "${HBASE_ENVIRONMENT_EXTRA_PATH}X" != "X" ] && [ -f ${HBASE_ENVIRONMENT_EXTRA_PATH} ]
then
    cat ${HBASE_ENVIRONMENT_EXTRA_PATH} >> ${post_hbaseenvsh}
else
    echo "ulimit -n ${openfilescount}" >> ${post_hbaseenvsh}
    echo "ulimit -u ${userprocessescount}" >> ${post_hbaseenvsh}
fi

cp ${pre_log4jproperties} ${post_log4jproperties}

# Need a copy of hdfs-site.xml for client defaults
cp ${HADOOP_CONF_DIR}/hdfs-site.xml ${HBASE_CONF_DIR}/hdfs-site.xml

exit 0