From 0e5657945de3494f6790284f13044bff5db4aa57 Mon Sep 17 00:00:00 2001 From: Christopher Tubbs Date: Thu, 17 Mar 2016 00:04:20 -0400 Subject: [PATCH] ACCUMULO-4162 Fix zookeeper version matching Apply many shellcheck fixes --- assemble/bin/LogForwarder.sh | 5 +- assemble/bin/accumulo | 52 ++++++----- assemble/bin/bootstrap_config.sh | 94 ++++++++++---------- assemble/bin/bootstrap_hdfs.sh | 16 ++-- assemble/bin/build_native_library.sh | 4 +- assemble/bin/check-slaves | 2 +- assemble/bin/config.sh | 61 +++++++------ assemble/bin/generate_monitor_certificate.sh | 26 +++--- assemble/bin/start-all.sh | 46 +++++----- assemble/bin/start-here.sh | 30 +++---- assemble/bin/start-server.sh | 39 ++++---- assemble/bin/stop-all.sh | 37 ++++---- assemble/bin/stop-here.sh | 21 ++--- assemble/bin/stop-server.sh | 22 ++--- assemble/bin/tdown.sh | 11 ++- assemble/bin/tool.sh | 24 ++--- assemble/bin/tup.sh | 12 ++- assemble/conf/templates/accumulo-env.sh | 4 +- core/src/main/scripts/generate-thrift.sh | 26 +++--- 19 files changed, 264 insertions(+), 268 deletions(-) diff --git a/assemble/bin/LogForwarder.sh b/assemble/bin/LogForwarder.sh index 5fe7871fc4b..0ebd9e49de7 100755 --- a/assemble/bin/LogForwarder.sh +++ b/assemble/bin/LogForwarder.sh @@ -36,15 +36,14 @@ # Start: Resolve Script Directory SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink +while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located done bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -script=$( basename "$SOURCE" ) # Stop: Resolve Script Directory . "$bin"/config.sh -${JAVA_HOME}/bin/java -cp $ACCUMULO_HOME/lib org.apache.accumulo.server.util.SendLogToChainsaw -d $ACCUMULO_LOG_DIR "$@" +"${JAVA_HOME}/bin/java" -cp "$ACCUMULO_HOME/lib" org.apache.accumulo.server.util.SendLogToChainsaw -d "$ACCUMULO_LOG_DIR" "$@" diff --git a/assemble/bin/accumulo b/assemble/bin/accumulo index 301d9a366ec..bb4fc797ec3 100755 --- a/assemble/bin/accumulo +++ b/assemble/bin/accumulo @@ -17,21 +17,20 @@ # Start: Resolve Script Directory SOURCE="${BASH_SOURCE[0]}" -while [ -h "${SOURCE}" ]; do # resolve $SOURCE until the file is no longer a symlink - bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )" +while [[ -h "${SOURCE}" ]]; do # resolve $SOURCE until the file is no longer a symlink + bin="$( cd -P "$(dirname "${SOURCE}")" && pwd )" SOURCE="$(readlink "${SOURCE}")" [[ "${SOURCE}" != /* ]] && SOURCE="${bin}/${SOURCE}" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located done -bin="$( cd -P "$( dirname "${SOURCE}" )" && pwd )" -script=$( basename "${SOURCE}" ) +bin="$(cd -P "$(dirname "${SOURCE}")" && pwd)" # Stop: Resolve Script Directory . "${bin}"/config.sh START_JAR="${ACCUMULO_HOME}/lib/accumulo-start.jar" -if [ ! -f "$START_JAR" ]; then - if [ -x /bin/rpm ]; then - START_JAR=$(echo $(/bin/rpm -E "%{_javadir}/accumulo/"accumulo-start-*.jar)) +if [[ ! -f "$START_JAR" ]]; then + if [[ -x /bin/rpm ]]; then + START_JAR=$(/bin/rpm -E "%{_javadir}/accumulo/"accumulo-start-*.jar) fi fi @@ -45,13 +44,13 @@ locationByProgram() echo "Cannot find '$1' and '$2' is not set in $ACCUMULO_CONF_DIR/accumulo-env.sh" exit 1 fi - while [ -h "${RESULT}" ]; do # resolve $RESULT until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$RESULT" )" && pwd )" + while [[ -h "${RESULT}" ]]; do # resolve $RESULT until the file is no longer a symlink + DIR="$( cd -P "$(dirname "$RESULT")" && pwd )" RESULT="$(readlink "${RESULT}")" [[ "${RESULT}" != /* ]] && RESULT="${DIR}/${RESULT}" # if $RESULT was a relative symlink, we need to resolve it relative to the path where the symlink file was located done # find the relative home directory, accounting for an extra bin directory - RESULT=$(dirname $(dirname "${RESULT}") ) + RESULT=$(dirname "$(dirname "${RESULT}")") echo "Auto-set ${2} to '${RESULT}'. To suppress this message, set ${2} in conf/accumulo-env.sh" eval "${2}=${RESULT}" } @@ -60,19 +59,17 @@ test -z "${JAVA_HOME}" && locationByProgram java JAVA_HOME test -z "${HADOOP_PREFIX}" && locationByProgram hadoop HADOOP_PREFIX test -z "${ZOOKEEPER_HOME}" && locationByProgram zkCli.sh ZOOKEEPER_HOME -DEFAULT_GENERAL_JAVA_OPTS="" - # # ACCUMULO_XTRAJARS is where all of the commandline -add items go into for reading by accumulo. # It also holds the JAR run with the jar command and, if possible, any items in the JAR manifest's Class-Path. # -if [ "$1" = "-add" ] ; then +if [[ "$1" == "-add" ]] ; then export ACCUMULO_XTRAJARS="$2" shift 2 else export ACCUMULO_XTRAJARS="" fi -if [ "$1" = "jar" -a -f "$2" ] ; then +if [[ "$1" == "jar" && -f "$2" ]] ; then if [[ $2 =~ ^/ ]]; then jardir="$(dirname "$2")" jarfile="$2" @@ -82,9 +79,9 @@ if [ "$1" = "jar" -a -f "$2" ] ; then fi if jar tf "$jarfile" | grep -q META-INF/MANIFEST.MF ; then cp="$(unzip -p "$jarfile" META-INF/MANIFEST.MF | grep ^Class-Path: | sed 's/^Class-Path: *//')" - if [ -n "$cp" ] ; then + if [[ -n "$cp" ]] ; then for j in $cp; do - if [ "$j" != "Class-Path:" ]; then + if [[ "$j" != "Class-Path:" ]]; then ACCUMULO_XTRAJARS="${jardir}/${j},$ACCUMULO_XTRAJARS" fi done @@ -96,11 +93,11 @@ fi # # Set up -D switches for JAAS and Kerberos if env variables set # -if [ ! -z ${ACCUMULO_JAAS_CONF} ]; then +if [[ ! -z ${ACCUMULO_JAAS_CONF} ]]; then ACCUMULO_GENERAL_OPTS="${ACCUMULO_GENERAL_OPTS} -Djava.security.auth.login.config=${ACCUMULO_JAAS_CONF}" fi -if [ ! -z ${ACCUMULO_KRB5_CONF} ]; then +if [[ ! -z ${ACCUMULO_KRB5_CONF} ]]; then ACCUMULO_GENERAL_OPTS="${ACCUMULO_GENERAL_OPTS} -Djava.security.krb5.conf=${ACCUMULO_KRB5_CONF}" fi @@ -120,26 +117,26 @@ LOG4J_JAR=$(find -H "${HADOOP_PREFIX}/lib" "${HADOOP_PREFIX}"/share/hadoop/commo # The `find` command could fail for environmental reasons or bad configuration # Avoid trying to run Accumulo when we can't find the jar -if [ -z "${LOG4J_JAR}" ]; then +if [[ -z "${LOG4J_JAR}" ]]; then echo "Could not locate Log4j jar in Hadoop installation at ${HADOOP_PREFIX}" exit 1 fi -CLASSPATH="${XML_FILES}:${START_JAR}:${LOG4J_JAR}" +CLASSPATH="${XML_FILES}:${START_JAR}:${LOG4J_JAR}:${CLASSPATH}" -if [ -z "${JAVA_HOME}" -o ! -d "${JAVA_HOME}" ]; then +if [[ -z "${JAVA_HOME}" || ! -d "${JAVA_HOME}" ]]; then echo "JAVA_HOME is not set or is not a directory. Please make sure it's set globally or in conf/accumulo-env.sh" exit 1 fi -if [ -z "${HADOOP_PREFIX}" -o ! -d "${HADOOP_PREFIX}" ]; then +if [[ -z "${HADOOP_PREFIX}" || ! -d "${HADOOP_PREFIX}" ]]; then echo "HADOOP_PREFIX is not set or is not a directory. Please make sure it's set globally or in conf/accumulo-env.sh" exit 1 fi -if [ -z "${ZOOKEEPER_HOME}" -o ! -d "${ZOOKEEPER_HOME}" ]; then +if [[ -z "${ZOOKEEPER_HOME}" || ! -d "${ZOOKEEPER_HOME}" ]]; then echo "ZOOKEEPER_HOME is not set or is not a directory. Please make sure it's set globally or in conf/accumulo-env.sh" exit 1 fi -if [ -z "${ACCUMULO_LOG_DIR}" ]; then +if [[ -z "${ACCUMULO_LOG_DIR}" ]]; then echo "ACCUMULO_LOG_DIR is not set. Please make sure it's set globally or in conf/accumulo-env.sh" exit 1 fi @@ -149,7 +146,7 @@ fi # explicitly in ${ACCUMULO_HOME}/conf/accumulo-env.sh # usually something like: # ${HADOOP_PREFIX}/lib/native/${PLATFORM} -if [ -e "${HADOOP_PREFIX}/lib/native/libhadoop.so" ]; then +if [[ -e "${HADOOP_PREFIX}/lib/native/libhadoop.so" ]]; then LIB_PATH="${HADOOP_PREFIX}/lib/native" LD_LIBRARY_PATH="${LIB_PATH}:${LD_LIBRARY_PATH}" # For Linux DYLD_LIBRARY_PATH="${LIB_PATH}:${DYLD_LIBRARY_PATH}" # For Mac @@ -160,9 +157,10 @@ fi export JAVA_HOME HADOOP_PREFIX ZOOKEEPER_HOME LD_LIBRARY_PATH DYLD_LIBRARY_PATH # # app isn't used anywhere, but it makes the process easier to spot when ps/top/snmp truncate the command line +ACCUMULO_OPTS_ARRAY=($ACCUMULO_OPTS) JAVA="${JAVA_HOME}/bin/java" -exec $JAVA "-Dapp=$1" \ - $ACCUMULO_OPTS \ +exec "$JAVA" "-Dapp=$1" \ + "${ACCUMULO_OPTS_ARRAY[@]}" \ -classpath "${CLASSPATH}" \ -XX:OnOutOfMemoryError="${ACCUMULO_KILL_CMD:-kill -9 %p}" \ -XX:-OmitStackTraceInFastThrow \ diff --git a/assemble/bin/bootstrap_config.sh b/assemble/bin/bootstrap_config.sh index 42b83ad135c..6b8bd18b392 100755 --- a/assemble/bin/bootstrap_config.sh +++ b/assemble/bin/bootstrap_config.sh @@ -31,7 +31,7 @@ EOF # Start: Resolve Script Directory SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink +while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located @@ -43,7 +43,7 @@ bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" # # Resolve accumulo home for bootstrapping # -ACCUMULO_HOME=$( cd -P ${bin}/.. && pwd ) +ACCUMULO_HOME=$( cd -P "${bin}/.." && pwd ) TEMPLATE_CONF_DIR="${ACCUMULO_HOME}/conf/templates" CONF_DIR="${ACCUMULO_HOME}/conf" ACCUMULO_SITE=accumulo-site.xml @@ -57,9 +57,9 @@ BASE_DIR= #Execute getopt if [[ $(uname -s) == "Linux" ]]; then - args=$(getopt -o "b:d:s:njov:h" -l "basedir:,dir:,size:,native,jvm,overwrite,version:,help" -q -- "$@") + args=($(getopt -o "b:d:s:njov:h" -l "basedir:,dir:,size:,native,jvm,overwrite,version:,help" -q -- "$@")) else # Darwin, BSD - args=$(getopt b:d:s:njov:h $*) + args=($(getopt b:d:s:njov:h "$@")) fi #Bad arguments @@ -67,7 +67,7 @@ if [[ $? != 0 ]]; then usage 1>&2 exit 1 fi -eval set -- $args +eval set -- "${args[@]}" for i do @@ -103,16 +103,16 @@ do esac done -while [[ "${OVERWRITE}" = "0" ]]; do +while [[ "${OVERWRITE}" == "0" ]]; do if [[ -e "${CONF_DIR}/${ACCUMULO_ENV}" || -e "${CONF_DIR}/${ACCUMULO_SITE}" ]]; then echo "Warning your current config files in ${CONF_DIR} will be overwritten!" echo echo "How would you like to proceed?:" select CHOICE in 'Continue with overwrite' 'Specify new conf dir'; do - if [[ "${CHOICE}" = 'Specify new conf dir' ]]; then + if [[ "${CHOICE}" == 'Specify new conf dir' ]]; then echo -n "Please specifiy new conf directory: " read CONF_DIR - elif [[ "${CHOICE}" = 'Continue with overwrite' ]]; then + elif [[ "${CHOICE}" == 'Continue with overwrite' ]]; then OVERWRITE=1 fi break @@ -233,13 +233,13 @@ if [[ -z "${HADOOP_VERSION}" ]]; then echo echo "Choose the Apache Hadoop version:" select HADOOP in 'Hadoop 1' 'Hadoop 2' 'HDP 2.0/2.1' 'HDP 2.2' ; do - if [ "${HADOOP}" == "Hadoop 2" ]; then + if [[ "${HADOOP}" == "Hadoop 2" ]]; then HADOOP_VERSION="2" - elif [ "${HADOOP}" == "Hadoop 1" ]; then + elif [[ "${HADOOP}" == "Hadoop 1" ]]; then HADOOP_VERSION="1" - elif [ "${HADOOP}" == "HDP 2.0/2.1" ]; then + elif [[ "${HADOOP}" == "HDP 2.0/2.1" ]]; then HADOOP_VERSION="HDP2" - elif [ "${HADOOP}" == "HDP 2.2" ]; then + elif [[ "${HADOOP}" == "HDP 2.2" ]]; then HADOOP_VERSION="HDP2.2" fi echo "Using Hadoop version '${HADOOP_VERSION}' configuration" @@ -279,13 +279,13 @@ if [[ ! -z "${BASE_DIR}" ]]; then fi #Configure accumulo-env.sh -mkdir -p "${CONF_DIR}" && cp ${TEMPLATE_CONF_DIR}/* ${CONF_DIR}/ +mkdir -p "${CONF_DIR}" && cp "${TEMPLATE_CONF_DIR}"/* "${CONF_DIR}/" sed -e "s/\${tServerHigh_tServerLow}/${!TSERVER}/" \ -e "s/\${masterHigh_masterLow}/${!MASTER}/" \ -e "s/\${monitorHigh_monitorLow}/${!MONITOR}/" \ -e "s/\${gcHigh_gcLow}/${!GC}/" \ -e "s/\${otherHigh_otherLow}/${!OTHER}/" \ - ${TEMPLATE_CONF_DIR}/$ACCUMULO_ENV > ${CONF_DIR}/$ACCUMULO_ENV + "${TEMPLATE_CONF_DIR}/$ACCUMULO_ENV" > "${CONF_DIR}/$ACCUMULO_ENV" #Configure accumulo-site.xml sed -e "s/\${memMapMax}/${!MEMORY_MAP_MAX}/" \ @@ -294,77 +294,77 @@ sed -e "s/\${memMapMax}/${!MEMORY_MAP_MAX}/" \ -e "s/\${cacheIndexSize}/${!CACHE_INDEX_SIZE}/" \ -e "s/\${sortBufferSize}/${!SORT_BUFFER_SIZE}/" \ -e "s/\${waLogMaxSize}/${!WAL_MAX_SIZE}/" \ - -e "s=\${mvnProjBaseDir}=${MAVEN_PROJ_BASEDIR}=" ${TEMPLATE_CONF_DIR}/$ACCUMULO_SITE > ${CONF_DIR}/$ACCUMULO_SITE + -e "s=\${mvnProjBaseDir}=${MAVEN_PROJ_BASEDIR}=" "${TEMPLATE_CONF_DIR}/$ACCUMULO_SITE" > "${CONF_DIR}/$ACCUMULO_SITE" #Configure for hadoop 1 -if [[ "$HADOOP_VERSION" = "1" ]]; then - sed -e 's/^test -z \"$HADOOP_CONF_DIR\"/#test -z \"$HADOOP_CONF_DIR\"/' -e 's/^# test -z "$HADOOP_CONF_DIR"/test -z \"$HADOOP_CONF_DIR\"/' ${CONF_DIR}/$ACCUMULO_ENV > temp - mv temp ${CONF_DIR}/$ACCUMULO_ENV +if [[ "$HADOOP_VERSION" == "1" ]]; then + sed -e 's/^test -z \"''$''HADOOP_CONF_DIR\"/#test -z \"''$''HADOOP_CONF_DIR\"/' -e 's/^# test -z "''$''HADOOP_CONF_DIR"/test -z \"''$''HADOOP_CONF_DIR\"/' "${CONF_DIR}/$ACCUMULO_ENV" > temp + mv temp "${CONF_DIR}/$ACCUMULO_ENV" sed -e 's///-->/' \ - ${CONF_DIR}/$ACCUMULO_SITE > temp - mv temp ${CONF_DIR}/$ACCUMULO_SITE + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" sed -e 's///-->/' \ - ${CONF_DIR}/$ACCUMULO_SITE > temp - mv temp ${CONF_DIR}/$ACCUMULO_SITE + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" sed -e 's///-->/' \ - ${CONF_DIR}/$ACCUMULO_SITE > temp - mv temp ${CONF_DIR}/$ACCUMULO_SITE + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" elif [[ "${HADOOP_VERSION}" == "2" ]]; then sed -e 's///-->/' \ - ${CONF_DIR}/$ACCUMULO_SITE > temp - mv temp ${CONF_DIR}/$ACCUMULO_SITE + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" sed -e 's///-->/' \ - ${CONF_DIR}/$ACCUMULO_SITE > temp - mv temp ${CONF_DIR}/$ACCUMULO_SITE + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" sed -e 's///-->/' \ - ${CONF_DIR}/$ACCUMULO_SITE > temp - mv temp ${CONF_DIR}/$ACCUMULO_SITE + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" elif [[ "${HADOOP_VERSION}" == "HDP2" ]]; then sed -e 's///-->/' \ - ${CONF_DIR}/$ACCUMULO_SITE > temp - mv temp ${CONF_DIR}/$ACCUMULO_SITE + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" sed -e 's///-->/' \ - ${CONF_DIR}/$ACCUMULO_SITE > temp - mv temp ${CONF_DIR}/$ACCUMULO_SITE + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" sed -e 's///-->/' \ - ${CONF_DIR}/$ACCUMULO_SITE > temp - mv temp ${CONF_DIR}/$ACCUMULO_SITE + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" elif [[ "${HADOOP_VERSION}" == "HDP2.2" ]]; then sed -e 's///-->/' \ - ${CONF_DIR}/$ACCUMULO_SITE > temp - mv temp ${CONF_DIR}/$ACCUMULO_SITE + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" sed -e 's///-->/' \ - ${CONF_DIR}/$ACCUMULO_SITE > temp - mv temp ${CONF_DIR}/$ACCUMULO_SITE + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" sed -e 's///-->/' \ - ${CONF_DIR}/$ACCUMULO_SITE > temp - mv temp ${CONF_DIR}/$ACCUMULO_SITE + "${CONF_DIR}/$ACCUMULO_SITE" > temp + mv temp "${CONF_DIR}/$ACCUMULO_SITE" fi #Additional setup steps for native configuration. -if [[ "${TYPE}" = "native" ]]; then - if [[ "$(uname)" = 'Linux' ]]; then +if [[ "${TYPE}" == "native" ]]; then + if [[ "$(uname)" == 'Linux' ]]; then if [[ -z $HADOOP_PREFIX ]]; then echo "HADOOP_PREFIX not set cannot automatically configure LD_LIBRARY_PATH" else - NATIVE_LIB=$(readlink -ef $(dirname $(for x in $(find $HADOOP_PREFIX -name libhadoop.so); do ld $x 2>/dev/null && echo $x && break; done) 2>>/dev/null) 2>>/dev/null) + NATIVE_LIB=$(readlink -e "$(dirname "$(find -H "$HADOOP_PREFIX" -name libhadoop.so -exec ld {} \; -print -quit 2>/dev/null)")") if [[ -z $NATIVE_LIB ]]; then echo -e "Native libraries could not be found for your sytem in: $HADOOP_PREFIX" else - sed "/# Should the monitor/ i export LD_LIBRARY_PATH=${NATIVE_LIB}:\${LD_LIBRARY_PATH}" ${CONF_DIR}/$ACCUMULO_ENV > temp - mv temp ${CONF_DIR}/$ACCUMULO_ENV + sed "/# Should the monitor/ i export LD_LIBRARY_PATH=${NATIVE_LIB}:\${LD_LIBRARY_PATH}" "${CONF_DIR}/$ACCUMULO_ENV" > temp + mv temp "${CONF_DIR}/$ACCUMULO_ENV" echo -e "Added ${NATIVE_LIB} to the LD_LIBRARY_PATH" fi fi diff --git a/assemble/bin/bootstrap_hdfs.sh b/assemble/bin/bootstrap_hdfs.sh index 6f38f633f47..66d1947ed29 100755 --- a/assemble/bin/bootstrap_hdfs.sh +++ b/assemble/bin/bootstrap_hdfs.sh @@ -17,7 +17,7 @@ # Start: Resolve Script Directory SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink +while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located @@ -32,7 +32,7 @@ bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" # SYSTEM_CONTEXT_HDFS_DIR=$(grep -A1 "general.vfs.classpaths" "$ACCUMULO_CONF_DIR/accumulo-site.xml" | tail -1 | perl -pe 's/\s+//; s/<\/value>//; s|[^/]+$||; print $ARGV[1]') -if [ -z "$SYSTEM_CONTEXT_HDFS_DIR" ] +if [[ -z "$SYSTEM_CONTEXT_HDFS_DIR" ]] then echo "Your accumulo-site.xml file is not set up for the HDFS Classloader. Please add the following to your accumulo-site.xml file:" echo "" @@ -48,7 +48,7 @@ fi # Create the system context directy in HDFS if it does not exist # "$HADOOP_PREFIX/bin/hadoop" fs -ls "$SYSTEM_CONTEXT_HDFS_DIR" > /dev/null -if [ $? -ne 0 ]; then +if [[ $? -ne 0 ]]; then "$HADOOP_PREFIX/bin/hadoop" fs -mkdir "$SYSTEM_CONTEXT_HDFS_DIR" > /dev/null fi @@ -61,7 +61,7 @@ NUM_SLAVES=$(egrep -v '(^#|^\s*$)' "$SLAVES" | wc -l) #let each datanode service around 50 clients let "REP=$NUM_SLAVES/50" -if [ $REP -lt 3 ]; then +if [[ $REP -lt 3 ]]; then REP=3 fi @@ -78,7 +78,7 @@ fi "$HADOOP_PREFIX/bin/hadoop" fs -rmr "$SYSTEM_CONTEXT_HDFS_DIR/commons-vfs2.jar" > /dev/null "$HADOOP_PREFIX/bin/hadoop" fs -copyToLocal "$SYSTEM_CONTEXT_HDFS_DIR/accumulo-start.jar" "$ACCUMULO_HOME/lib/." > /dev/null "$HADOOP_PREFIX/bin/hadoop" fs -rmr "$SYSTEM_CONTEXT_HDFS_DIR/accumulo-start.jar" > /dev/null -for f in `cat $ACCUMULO_CONF_DIR/slaves` -do - rsync -ra --delete $ACCUMULO_HOME `dirname $ACCUMULO_HOME` -done + +while IFS=$' \t\n' read -r host; do + rsync -ra --delete "$ACCUMULO_HOME" "$host:$(readlink "$(dirname "$ACCUMULO_HOME")")" +done < <(egrep -v '^(\s*#.*|\s*)$' "$ACCUMULO_CONF_DIR/slaves") diff --git a/assemble/bin/build_native_library.sh b/assemble/bin/build_native_library.sh index c2531ee2403..02bb9cfd078 100755 --- a/assemble/bin/build_native_library.sh +++ b/assemble/bin/build_native_library.sh @@ -23,7 +23,6 @@ while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a sym [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located done bin=$( cd -P "$( dirname "$SOURCE" )" && pwd ) -script=$( basename "$SOURCE" ) # Stop: Resolve Script Directory @@ -56,7 +55,8 @@ native_dir=$(find "${TMP_DIR}" -maxdepth 1 -mindepth 1 -type d) cd "${native_dir}" || exit 1 # Make the native library -export USERFLAGS="$@" +# can't export arrays, flatten and parse in the Makefile +export USERFLAGS="$*" make # Make sure it didn't fail diff --git a/assemble/bin/check-slaves b/assemble/bin/check-slaves index 2af7f42ac51..457e9137f73 100755 --- a/assemble/bin/check-slaves +++ b/assemble/bin/check-slaves @@ -22,7 +22,7 @@ # the physical memory is the same # the mounts are the same on each machine # a set of writable locations (typically different disks) are in fact writable -# +# # In order to check for writable partitions, you must configure the WRITABLE variable below. # diff --git a/assemble/bin/config.sh b/assemble/bin/config.sh index 9bee91bf366..fb9208ee3bb 100755 --- a/assemble/bin/config.sh +++ b/assemble/bin/config.sh @@ -43,38 +43,37 @@ # ACCUMULO_JAAS_CONF Location of jaas.conf file. Needed by JAAS for things like Kerberos based logins # ACCUMULO_KRB5_CONF Location of krb5.conf file. Needed by Kerberos subsystems to find login servers -if [ -z "${ACCUMULO_HOME}" ] ; then +if [[ -z "${ACCUMULO_HOME}" ]] ; then # Start: Resolve Script Directory SOURCE="${BASH_SOURCE[0]}" - while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located done bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - script=$( basename "$SOURCE" ) # Stop: Resolve Script Directory - ACCUMULO_HOME=$( cd -P ${bin}/.. && pwd ) + ACCUMULO_HOME=$( cd -P "${bin}/.." && pwd ) export ACCUMULO_HOME fi -if [ ! -d "${ACCUMULO_HOME}" ]; then +if [[ ! -d "${ACCUMULO_HOME}" ]]; then echo "ACCUMULO_HOME=${ACCUMULO_HOME} is not a valid directory. Please make sure it exists" exit 1 fi ACCUMULO_CONF_DIR="${ACCUMULO_CONF_DIR:-$ACCUMULO_HOME/conf}" export ACCUMULO_CONF_DIR -if [ -z "$ACCUMULO_CONF_DIR" -o ! -d "$ACCUMULO_CONF_DIR" ] +if [[ -z "$ACCUMULO_CONF_DIR" || ! -d "$ACCUMULO_CONF_DIR" ]] then echo "ACCUMULO_CONF_DIR=$ACCUMULO_CONF_DIR is not a valid directory. Please make sure it exists" exit 1 fi -if [ -f $ACCUMULO_CONF_DIR/accumulo-env.sh ] ; then - . $ACCUMULO_CONF_DIR/accumulo-env.sh -elif [ -z "$ACCUMULO_TEST" ] ; then +if [[ -f $ACCUMULO_CONF_DIR/accumulo-env.sh ]]; then + . "$ACCUMULO_CONF_DIR/accumulo-env.sh" +elif [[ -z "$ACCUMULO_TEST" ]]; then # # Attempt to bootstrap configuration and continue # @@ -87,28 +86,28 @@ elif [ -z "$ACCUMULO_TEST" ] ; then exit 1 fi -if [ -z ${ACCUMULO_LOG_DIR} ]; then +if [[ -z ${ACCUMULO_LOG_DIR} ]]; then ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs fi -if [ -z "${ACCUMULO_VERIFY_ONLY}" ] ; then - mkdir -p $ACCUMULO_LOG_DIR 2>/dev/null +if [[ -z "${ACCUMULO_VERIFY_ONLY}" ]] ; then + mkdir -p "$ACCUMULO_LOG_DIR" 2>/dev/null fi export ACCUMULO_LOG_DIR -if [ -z "$HADOOP_PREFIX" ] +if [[ -z "$HADOOP_PREFIX" ]] then - HADOOP_PREFIX="`which hadoop`" - if [ -z "$HADOOP_PREFIX" ] + HADOOP_PREFIX="$(which hadoop)" + if [[ -z "$HADOOP_PREFIX" ]] then echo "You must set HADOOP_PREFIX" exit 1 fi - HADOOP_PREFIX=`dirname $HADOOP_PREFIX` - HADOOP_PREFIX=`dirname $HADOOP_PREFIX` + HADOOP_PREFIX=$(dirname "$HADOOP_PREFIX") + HADOOP_PREFIX=$(dirname "$HADOOP_PREFIX") fi -if [ ! -d "$HADOOP_PREFIX" ] +if [[ ! -d "$HADOOP_PREFIX" ]] then echo "HADOOP_PREFIX, which has a value of $HADOOP_PREFIX, is not a directory." exit 1 @@ -120,18 +119,18 @@ if [[ -f "$ACCUMULO_CONF_DIR/masters" ]]; then MASTER1=$(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/masters" | head -1) fi -if [ -z "${MONITOR}" ] ; then +if [[ -z "${MONITOR}" ]] ; then MONITOR=$MASTER1 - if [ -f "$ACCUMULO_CONF_DIR/monitor" ]; then + if [[ -f "$ACCUMULO_CONF_DIR/monitor" ]]; then MONITOR=$(egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/monitor" | head -1) fi - if [ -z "${MONITOR}" ] ; then + if [[ -z "${MONITOR}" ]] ; then echo "Could not infer a Monitor role. You need to either define the MONITOR env variable, define \"${ACCUMULO_CONF_DIR}/monitor\", or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." exit 1 fi fi -if [ ! -f "$ACCUMULO_CONF_DIR/tracers" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then - if [ -z "${MASTER1}" ] ; then +if [[ ! -f "$ACCUMULO_CONF_DIR/tracers" && -z "${ACCUMULO_VERIFY_ONLY}" ]]; then + if [[ -z "${MASTER1}" ]]; then echo "Could not find a master node to use as a default for the tracer role. Either set up \"${ACCUMULO_CONF_DIR}/tracers\" or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." exit 1 else @@ -140,8 +139,8 @@ if [ ! -f "$ACCUMULO_CONF_DIR/tracers" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then fi -if [ ! -f "$ACCUMULO_CONF_DIR/gc" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then - if [ -z "${MASTER1}" ] ; then +if [[ ! -f "$ACCUMULO_CONF_DIR/gc" && -z "${ACCUMULO_VERIFY_ONLY}" ]]; then + if [[ -z "${MASTER1}" ]] ; then echo "Could not infer a GC role. You need to either set up \"${ACCUMULO_CONF_DIR}/gc\" or make sure \"${ACCUMULO_CONF_DIR}/masters\" is non-empty." exit 1 else @@ -149,7 +148,7 @@ if [ ! -f "$ACCUMULO_CONF_DIR/gc" -a -z "${ACCUMULO_VERIFY_ONLY}" ]; then fi fi -NUMA=`which numactl 2>/dev/null` +NUMA=$(which numactl 2>/dev/null) NUMACTL_EXISTS="$?" NUMACTL_ARGS="--interleave=all" if [[ ${NUMACTL_EXISTS} -eq 0 ]] ; then @@ -158,7 +157,7 @@ else export NUMA_CMD="" fi -SSH='ssh -qnf -o ConnectTimeout=2' +export SSH='ssh -qnf -o ConnectTimeout=2' export HADOOP_HOME=$HADOOP_PREFIX export HADOOP_HOME_WARN_SUPPRESS=true @@ -170,15 +169,15 @@ export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-1} export ACCUMULO_MONITOR_BIND_ALL=${ACCUMULO_MONITOR_BIND_ALL:-"false"} # Check for jaas.conf configuration -if [ -z ${ACCUMULO_JAAS_CONF} ]; then - if [ -f ${ACCUMULO_CONF_DIR}/jaas.conf ]; then +if [[ -z ${ACCUMULO_JAAS_CONF} ]]; then + if [[ -f ${ACCUMULO_CONF_DIR}/jaas.conf ]]; then export ACCUMULO_JAAS_CONF=${ACCUMULO_CONF_DIR}/jaas.conf fi fi # Check for krb5.conf configuration -if [ -z ${ACCUMULO_KRB5_CONF} ]; then - if [ -f ${ACCUMULO_CONF_DIR}/krb5.conf ]; then +if [[ -z ${ACCUMULO_KRB5_CONF} ]]; then + if [[ -f ${ACCUMULO_CONF_DIR}/krb5.conf ]]; then export ACCUMULO_KRB5_CONF=${ACCUMULO_CONF_DIR}/krb5.conf fi fi diff --git a/assemble/bin/generate_monitor_certificate.sh b/assemble/bin/generate_monitor_certificate.sh index edd415912bb..eaf3c4854ab 100755 --- a/assemble/bin/generate_monitor_certificate.sh +++ b/assemble/bin/generate_monitor_certificate.sh @@ -17,7 +17,7 @@ # Start: Resolve Script Directory SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink +while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located @@ -34,33 +34,33 @@ KEYSTOREPATH="$ACCUMULO_HOME/conf/keystore.jks" TRUSTSTOREPATH="$ACCUMULO_HOME/conf/cacerts.jks" CERTPATH="$ACCUMULO_HOME/conf/server.cer" -if [ -e "$KEYSTOREPATH" ]; then - rm -i $KEYSTOREPATH - if [ -e "$KEYSTOREPATH" ]; then +if [[ -e "$KEYSTOREPATH" ]]; then + rm -i "$KEYSTOREPATH" + if [[ -e "$KEYSTOREPATH" ]]; then echo "KeyStore already exists, exiting" exit 1 fi fi -if [ -e "$TRUSTSTOREPATH" ]; then - rm -i $TRUSTSTOREPATH - if [ -e "$TRUSTSTOREPATH" ]; then +if [[ -e "$TRUSTSTOREPATH" ]]; then + rm -i "$TRUSTSTOREPATH" + if [[ -e "$TRUSTSTOREPATH" ]]; then echo "TrustStore already exists, exiting" exit 2 fi fi -if [ -e "$CERTPATH" ]; then - rm -i $CERTPATH - if [ -e "$CERTPATH" ]; then +if [[ -e "$CERTPATH" ]]; then + rm -i "$CERTPATH" + if [[ -e "$CERTPATH" ]]; then echo "Certificate already exists, exiting" exit 3 fi fi -${JAVA_HOME}/bin/keytool -genkey -alias $ALIAS -keyalg RSA -keypass $KEYPASS -storepass $KEYPASS -keystore $KEYSTOREPATH -${JAVA_HOME}/bin/keytool -export -alias $ALIAS -storepass $KEYPASS -file $CERTPATH -keystore $KEYSTOREPATH -echo "yes" | ${JAVA_HOME}/bin/keytool -import -v -trustcacerts -alias $ALIAS -file $CERTPATH -keystore $TRUSTSTOREPATH -storepass $STOREPASS +"${JAVA_HOME}/bin/keytool" -genkey -alias "$ALIAS" -keyalg RSA -keypass "$KEYPASS" -storepass "$KEYPASS" -keystore "$KEYSTOREPATH" +"${JAVA_HOME}/bin/keytool" -export -alias "$ALIAS" -storepass "$KEYPASS" -file "$CERTPATH" -keystore "$KEYSTOREPATH" +echo "yes" | "${JAVA_HOME}/bin/keytool" -import -v -trustcacerts -alias "$ALIAS" -file "$CERTPATH" -keystore "$TRUSTSTOREPATH" -storepass "$STOREPASS" echo echo "keystore and truststore generated. now add the following to accumulo-site.xml:" diff --git a/assemble/bin/start-all.sh b/assemble/bin/start-all.sh index b71ba705889..59594ae4d9f 100755 --- a/assemble/bin/start-all.sh +++ b/assemble/bin/start-all.sh @@ -17,7 +17,7 @@ # Start: Resolve Script Directory SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink +while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located @@ -28,50 +28,52 @@ bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" . "$bin"/config.sh unset DISPLAY -if [ ! -f $ACCUMULO_CONF_DIR/accumulo-env.sh ] ; then +if [[ ! -f $ACCUMULO_CONF_DIR/accumulo-env.sh ]] ; then echo "${ACCUMULO_CONF_DIR}/accumulo-env.sh does not exist. Please make sure you configure Accumulo before you run anything" echo "We provide examples you can copy in ${ACCUMULO_HOME}/conf/examples/ which are set up for your memory footprint" exit 1 fi -if [ -z "$ZOOKEEPER_HOME" ] ; then +if [[ -z "$ZOOKEEPER_HOME" ]] ; then echo "ZOOKEEPER_HOME is not set. Please make sure it's set globally or in conf/accumulo-env.sh" exit 1 fi -if [ ! -d $ZOOKEEPER_HOME ]; then +if [[ ! -d $ZOOKEEPER_HOME ]]; then echo "ZOOKEEPER_HOME is not a directory: $ZOOKEEPER_HOME" echo "Please check the setting, either globally or in accumulo-env.sh." exit 1 fi -ZOOKEEPER_VERSION=$(find -L $ZOOKEEPER_HOME -maxdepth 1 -name "zookeeper-[0-9]*.jar" | head -1) -if [ -z "$ZOOKEEPER_VERSION" ]; then +ZOOKEEPER_VERSION=$(find -L "$ZOOKEEPER_HOME" -maxdepth 1 -name "zookeeper-[0-9]*.jar" | head -1) +if [[ -z "$ZOOKEEPER_VERSION" ]]; then echo "A Zookeeper JAR was not found in $ZOOKEEPER_HOME." echo "Please check ZOOKEEPER_HOME, either globally or in accumulo-env.sh." exit 1 fi -ZOOKEEPER_VERSION=${ZOOKEEPER_VERSION##$ZOOKEEPER_HOME/zookeeper-} -ZOOKEEPER_VERSION=${ZOOKEEPER_VERSION%%.jar} +ZOOKEEPER_VERSION=$(basename "${ZOOKEEPER_VERSION##*-}" .jar) -if [ "$ZOOKEEPER_VERSION" '<' "3.3.0" ]; then +if [[ "$ZOOKEEPER_VERSION" < "3.3.0" ]]; then echo "WARN : Using Zookeeper $ZOOKEEPER_VERSION. Use version 3.3.0 or greater to avoid zookeeper deadlock bug."; fi -${bin}/start-server.sh $MONITOR monitor +"${bin}/start-server.sh" "$MONITOR" monitor -if [ "$1" != "--notSlaves" ]; then - ${bin}/tup.sh +if [[ "$1" != "--notSlaves" ]]; then + "${bin}/tup.sh" fi -${bin}/accumulo org.apache.accumulo.master.state.SetGoalState NORMAL -for master in `egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/masters"`; do - ${bin}/start-server.sh $master master -done +"${bin}/accumulo" org.apache.accumulo.master.state.SetGoalState NORMAL -for gc in `egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/gc"`; do - ${bin}/start-server.sh $gc gc "garbage collector" -done +startServersFromHostsFile() { + # use hostfile in conf dir to get hosts, and start each server with the remaining args + local hostfile; hostfile="$1" + shift + local otherArgs; otherArgs=("$@") + while IFS=$' \t\n' read -r host; do + "${bin}/start-server.sh" "$host" "${otherArgs[@]}" + done < <(egrep -v '^(\s*#.*|\s*)$' "$ACCUMULO_CONF_DIR/$hostfile") +} -for tracer in `egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/tracers"`; do - ${bin}/start-server.sh $tracer tracer -done +startServersFromHostsFile masters master +startServersFromHostsFile gc gc 'garbage collector' +startServersFromHostsFile tracers tracer diff --git a/assemble/bin/start-here.sh b/assemble/bin/start-here.sh index 9abdb387273..3d2d3c55b56 100755 --- a/assemble/bin/start-here.sh +++ b/assemble/bin/start-here.sh @@ -21,7 +21,7 @@ # Start: Resolve Script Directory SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink +while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located @@ -32,47 +32,47 @@ bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" . "$bin"/config.sh IFCONFIG=/sbin/ifconfig -if [ ! -x $IFCONFIG ]; then +if [[ ! -x $IFCONFIG ]]; then IFCONFIG='/bin/netstat -ie' fi -ip=$($IFCONFIG 2>/dev/null| grep inet[^6] | awk '{print $2}' | sed 's/addr://' | grep -v 0.0.0.0 | grep -v 127.0.0.1 | head -n 1) -if [ $? != 0 ]; then +ip=$($IFCONFIG 2>/dev/null| grep 'inet[^6]' | awk '{print $2}' | sed 's/addr://' | grep -v '0[.]0[.]0[.]0' | grep -v '127[.]0[.]0[.]1' | head -n 1) +if [[ $? != 0 ]]; then ip=$(python -c 'import socket as s; print s.gethostbyname(s.getfqdn())') fi HOSTS="$(hostname -a 2> /dev/null) $(hostname) localhost 127.0.0.1 $ip" for host in $HOSTS; do - if grep -q "^${host}\$" $ACCUMULO_CONF_DIR/slaves; then - ${bin}/start-server.sh $host tserver "tablet server" + if grep -q "^${host}\$" "$ACCUMULO_CONF_DIR/slaves"; then + "${bin}/start-server.sh" "$host" tserver "tablet server" break fi done for host in $HOSTS; do - if grep -q "^${host}\$" $ACCUMULO_CONF_DIR/masters; then - ${bin}/accumulo org.apache.accumulo.master.state.SetGoalState NORMAL - ${bin}/start-server.sh $host master + if grep -q "^${host}\$" "$ACCUMULO_CONF_DIR/masters"; then + "${bin}/accumulo" org.apache.accumulo.master.state.SetGoalState NORMAL + "${bin}/start-server.sh" "$host" master break fi done for host in $HOSTS; do - if grep -q "^${host}\$" $ACCUMULO_CONF_DIR/gc; then - ${bin}/start-server.sh $host gc "garbage collector" + if grep -q "^${host}\$" "$ACCUMULO_CONF_DIR/gc"; then + "${bin}/start-server.sh" "$host" gc "garbage collector" break fi done for host in $HOSTS; do - if [ ${host} = "${MONITOR}" ]; then - ${bin}/start-server.sh $MONITOR monitor + if [[ ${host} == "${MONITOR}" ]]; then + "${bin}/start-server.sh" "$MONITOR" monitor break fi done for host in $HOSTS; do - if grep -q "^${host}\$" $ACCUMULO_CONF_DIR/tracers; then - ${bin}/start-server.sh $host tracer + if grep -q "^${host}\$" "$ACCUMULO_CONF_DIR/tracers"; then + "${bin}/start-server.sh" "$host" tracer break fi done diff --git a/assemble/bin/start-server.sh b/assemble/bin/start-server.sh index 8f37d50f210..991f7327db8 100755 --- a/assemble/bin/start-server.sh +++ b/assemble/bin/start-server.sh @@ -17,20 +17,19 @@ # Start: Resolve Script Directory SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink +while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located done bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -script=$( basename "$SOURCE" ) # Stop: Resolve Script Directory . "$bin"/config.sh HOST="$1" host "$1" >/dev/null 2>/dev/null -if [ $? -ne 0 ]; then +if [[ $? -ne 0 ]]; then LOGHOST="$1" else LOGHOST=$(host "$1" | head -1 | cut -d' ' -f1) @@ -38,51 +37,51 @@ fi ADDRESS="$1" SERVICE="$2" LONGNAME="$3" -if [ -z "$LONGNAME" ]; then +if [[ -z "$LONGNAME" ]]; then LONGNAME="$2" fi -SLAVES=$( wc -l < ${ACCUMULO_CONF_DIR}/slaves ) +SLAVES=$( wc -l < "${ACCUMULO_CONF_DIR}/slaves" ) IFCONFIG=/sbin/ifconfig -if [ ! -x $IFCONFIG ]; then +if [[ ! -x $IFCONFIG ]]; then IFCONFIG='/bin/netstat -ie' fi -ip=$($IFCONFIG 2>/dev/null| grep inet[^6] | awk '{print $2}' | sed 's/addr://' | grep -v 0.0.0.0 | grep -v 127.0.0.1 | head -n 1) -if [ $? != 0 ] +ip=$($IFCONFIG 2>/dev/null| grep 'inet[^6]' | awk '{print $2}' | sed 's/addr://' | grep -v '0[.]0[.]0[.]0' | grep -v '127[.]0[.]0[.]1' | head -n 1) +if [[ $? != 0 ]] then ip=$(python -c 'import socket as s; print s.gethostbyname(s.getfqdn())') fi # When the hostname provided is the alias/shortname, try to use the FQDN to make # sure we send the right address to the Accumulo process. -if [ "$HOST" = "`hostname -s`" ]; then - HOST="`hostname -f`" +if [[ "$HOST" == "$(hostname -s)" ]]; then + HOST="$(hostname -f)" ADDRESS="$HOST" fi # ACCUMULO-1985 Allow monitor to bind on all interfaces -if [ ${SERVICE} == "monitor" -a ${ACCUMULO_MONITOR_BIND_ALL} == "true" ]; then +if [[ ${SERVICE} == "monitor" && ${ACCUMULO_MONITOR_BIND_ALL} == "true" ]]; then ADDRESS="0.0.0.0" fi -if [ "$HOST" = "localhost" -o "$HOST" = "`hostname -f`" -o "$HOST" = "$ip" ]; then - PID=$(ps -ef | egrep ${ACCUMULO_HOME}/.*/accumulo.*.jar | grep "Main $SERVICE" | grep -v grep | awk {'print $2'} | head -1) +if [[ "$HOST" == "localhost" || "$HOST" == "$(hostname -f)" || "$HOST" == "$ip" ]]; then + PID=$(ps -ef | egrep "${ACCUMULO_HOME}"/.*/accumulo.*.jar | grep "Main $SERVICE" | grep -v grep | awk '{print $2}' | head -1) else - PID=$($SSH $HOST ps -ef | egrep ${ACCUMULO_HOME}/.*/accumulo.*.jar | grep "Main $SERVICE" | grep -v grep | awk {'print $2'} | head -1) + PID=$($SSH "$HOST" ps -ef | egrep "${ACCUMULO_HOME}"/.*/accumulo.*.jar | grep "Main $SERVICE" | grep -v grep | awk '{print $2}' | head -1) fi -if [ -z "$PID" ]; then +if [[ -z "$PID" ]]; then echo "Starting $LONGNAME on $HOST" - if [ "$HOST" = "localhost" -o "$HOST" = "`hostname -f`" -o "$HOST" = "$ip" ]; then - nohup ${NUMA_CMD} ${bin}/accumulo ${SERVICE} --address ${ADDRESS} >${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out 2>${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err & + if [[ "$HOST" == "localhost" || "$HOST" == "$(hostname -f)" || "$HOST" == "$ip" ]]; then + nohup "${NUMA_CMD}" "${bin}/accumulo" "${SERVICE}" --address "${ADDRESS}" >"${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out" 2>"${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err" & MAX_FILES_OPEN=$(ulimit -n) else - $SSH $HOST "bash -c 'exec nohup ${NUMA_CMD} ${bin}/accumulo ${SERVICE} --address ${ADDRESS} >${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out 2>${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err' &" - MAX_FILES_OPEN=$($SSH $HOST "/usr/bin/env bash -c 'ulimit -n'") + $SSH "$HOST" "bash -c 'exec nohup \"${NUMA_CMD}\" \"${bin}/accumulo\" \"${SERVICE}\" --address \"${ADDRESS}\" >\"${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.out\" 2>\"${ACCUMULO_LOG_DIR}/${SERVICE}_${LOGHOST}.err\"' &" + MAX_FILES_OPEN=$($SSH "$HOST" "/usr/bin/env bash -c 'ulimit -n'") fi - if [ -n "$MAX_FILES_OPEN" ] && [ -n "$SLAVES" ] ; then + if [[ -n "$MAX_FILES_OPEN" ]] && [[ -n "$SLAVES" ]] ; then MAX_FILES_RECOMMENDED=${MAX_FILES_RECOMMENDED:-32768} if (( SLAVES > 10 )) && (( MAX_FILES_OPEN < MAX_FILES_RECOMMENDED )) then diff --git a/assemble/bin/stop-all.sh b/assemble/bin/stop-all.sh index 3bd30bd758a..32a0c4729c5 100755 --- a/assemble/bin/stop-all.sh +++ b/assemble/bin/stop-all.sh @@ -18,7 +18,7 @@ # Start: Resolve Script Directory SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink +while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located @@ -29,9 +29,9 @@ bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" . "$bin"/config.sh echo "Stopping accumulo services..." -${bin}/accumulo admin "$@" stopAll +"${bin}/accumulo" admin "$@" stopAll -if [ $? -ne 0 ]; then +if [[ $? -ne 0 ]]; then echo "Invalid password or unable to connect to the master" echo "Initiating forced shutdown in 15 seconds (Ctrl-C to abort)" sleep 10 @@ -43,26 +43,27 @@ fi sleep 5 +stopServersFromHostsFile() { + # use hostfile in conf dir to get hosts, and start each server with the remaining args + local hostfile; hostfile="$1" + shift + local otherArgs; otherArgs=("$@") + while IFS=$' \t\n' read -r host; do + "${bin}/stop-server.sh" "$host" "$ACCUMULO_HOME/lib/accumulo-start.*.jar" "${otherArgs[@]}" + done < <(egrep -v '^(\s*#.*|\s*)$' "$ACCUMULO_CONF_DIR/$hostfile") +} + #look for master and gc processes not killed by 'admin stopAll' for signal in TERM KILL ; do - for master in `grep -v '^#' "$ACCUMULO_CONF_DIR/masters"`; do - ${bin}/stop-server.sh $master "$ACCUMULO_HOME/lib/accumulo-start.*.jar" master $signal - done - - for gc in `grep -v '^#' "$ACCUMULO_CONF_DIR/gc"`; do - ${bin}/stop-server.sh "$gc" "$ACCUMULO_HOME/lib/accumulo-start.*.jar" gc $signal - done - - ${bin}/stop-server.sh "$MONITOR" "$ACCUMULO_HOME/.*/accumulo-start.*.jar" monitor $signal - - for tracer in `egrep -v '(^#|^\s*$)' "$ACCUMULO_CONF_DIR/tracers"`; do - ${bin}/stop-server.sh $tracer "$ACCUMULO_HOME/.*/accumulo-start.*.jar" tracer $signal - done + stopServersFromHostsFile masters master $signal + stopServersFromHostsFile gc gc $signal + "${bin}/stop-server.sh" "$MONITOR" "$ACCUMULO_HOME/.*/accumulo-start.*.jar" monitor $signal + stopServersFromHostsFile tracers tracer $signal done # stop tserver still running -${bin}/tdown.sh +"${bin}/tdown.sh" echo "Cleaning all server entries in ZooKeeper" -$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.util.ZooZap -master -tservers -tracers +"$ACCUMULO_HOME/bin/accumulo" org.apache.accumulo.server.util.ZooZap -master -tservers -tracers diff --git a/assemble/bin/stop-here.sh b/assemble/bin/stop-here.sh index 49defa9b7cf..9510c98eee6 100755 --- a/assemble/bin/stop-here.sh +++ b/assemble/bin/stop-here.sh @@ -21,7 +21,7 @@ # Start: Resolve Script Directory SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink +while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located @@ -35,26 +35,27 @@ ACCUMULO="$ACCUMULO_HOME/lib/accumulo-start.jar" # Determine hostname without errors to user HOSTNAME=$(hostname -a 2> /dev/null | head -1) -if [ -z ${HOSTNAME} ]; then +if [[ -z ${HOSTNAME} ]]; then HOSTNAME=$(hostname) fi -if egrep -q localhost\|127.0.0.1 $ACCUMULO_CONF_DIR/slaves; then - $bin/accumulo admin stop localhost +if egrep -q 'localhost|127.0.0.1' "$ACCUMULO_CONF_DIR/slaves"; then + "$bin/accumulo" admin stop localhost else - for host in "$(hostname -a 2> /dev/null) $(hostname)"; do - if grep -q ${host} $ACCUMULO_CONF_DIR/slaves; then - ${bin}/accumulo admin stop $host + hostnames=("$(hostname -a 2> /dev/null)" "$(hostname)") + for host in "${hostnames[@]}"; do + if grep -q "${host}" "$ACCUMULO_CONF_DIR/slaves"; then + "${bin}/accumulo" admin stop "$host" fi done fi for signal in TERM KILL; do for svc in tserver gc master monitor tracer; do - PID=$(ps -ef | egrep ${ACCUMULO} | grep "Main $svc" | grep -v grep | grep -v stop-here.sh | awk '{print $2}' | head -1) - if [ ! -z $PID ]; then + PID=$(ps -ef | egrep "${ACCUMULO}" | grep "Main $svc" | grep -v grep | grep -v stop-here.sh | awk '{print $2}' | head -1) + if [[ ! -z $PID ]]; then echo "Stopping ${svc} on ${HOSTNAME} with signal ${signal}" - kill -s ${signal} ${PID} + kill -s "${signal}" "${PID}" fi done done diff --git a/assemble/bin/stop-server.sh b/assemble/bin/stop-server.sh index d615172dfd8..e6ffae3e7a2 100755 --- a/assemble/bin/stop-server.sh +++ b/assemble/bin/stop-server.sh @@ -17,7 +17,7 @@ # Start: Resolve Script Directory SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink +while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located @@ -30,27 +30,27 @@ bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" HOST=$1 IFCONFIG=/sbin/ifconfig -if [ ! -x $IFCONFIG ] +if [[ ! -x $IFCONFIG ]] then IFCONFIG='/bin/netstat -ie' fi -ip=$($IFCONFIG 2>/dev/null| grep inet[^6] | awk '{print $2}' | sed 's/addr://' | grep -v 0.0.0.0 | grep -v 127.0.0.1 | head -n 1) -if [ $? != 0 ] +ip=$($IFCONFIG 2>/dev/null| grep 'inet[^6]' | awk '{print $2}' | sed 's/addr://' | grep -v '0[.]0[.]0[.]0' | grep -v '127[.]0[.]0[.]1' | head -n 1) +if [[ $? != 0 ]] then ip=$(python -c 'import socket as s; print s.gethostbyname(s.getfqdn())') fi # only stop if there's not one already running -if [ "$HOST" = "localhost" -o "$HOST" = "`hostname -s`" -o "$HOST" = "`hostname -f`" -o "$HOST" = "$ip" ]; then - PID=$(ps -ef | grep "$ACCUMULO_HOME" | egrep ${2} | grep "Main ${3}" | grep -v grep | grep -v ssh | grep -v stop-server.sh | awk {'print $2'} | head -1) - if [ ! -z $PID ]; then +if [[ "$HOST" == "localhost" || "$HOST" == "$(hostname -s)" || "$HOST" == "$(hostname -f)" || "$HOST" == "$ip" ]]; then + PID=$(ps -ef | grep "$ACCUMULO_HOME" | egrep "${2}" | grep "Main ${3}" | grep -v grep | grep -v ssh | grep -v stop-server.sh | awk '{print $2}' | head -1) + if [[ ! -z $PID ]]; then echo "Stopping ${3} on $1"; - kill -s ${4} ${PID} 2>/dev/null + kill -s "${4}" "${PID}" 2>/dev/null fi; else - PID=$(ssh -q -o 'ConnectTimeout 8' $1 "ps -ef | grep \"$ACCUMULO_HOME\" | egrep '${2}' | grep 'Main ${3}' | grep -v grep | grep -v ssh | grep -v stop-server.sh" | awk {'print $2'} | head -1) - if [ ! -z $PID ]; then + PID=$(ssh -q -o 'ConnectTimeout 8' "$1" "ps -ef | grep \"""$ACCUMULO_HOME""\" | egrep '${2}' | grep 'Main ${3}' | grep -v grep | grep -v ssh | grep -v stop-server.sh" | awk '{print $2}' | head -1) + if [[ ! -z $PID ]]; then echo "Stopping ${3} on $1"; - ssh -q -o 'ConnectTimeout 8' $1 "kill -s ${4} ${PID} 2>/dev/null" + ssh -q -o 'ConnectTimeout 8' "$1" "kill -s ""${4}"" ""${PID}"" 2>/dev/null" fi; fi diff --git a/assemble/bin/tdown.sh b/assemble/bin/tdown.sh index 141ad24497d..6d4c45c36a3 100755 --- a/assemble/bin/tdown.sh +++ b/assemble/bin/tdown.sh @@ -17,7 +17,7 @@ # Start: Resolve Script Directory SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink +while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located @@ -27,14 +27,13 @@ bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" . "$bin"/config.sh -HADOOP_CMD=$HADOOP_PREFIX/bin/hadoop -SLAVES=$ACCUMULO_CONF_DIR/slaves +SLAVES="$ACCUMULO_CONF_DIR/slaves" SLAVE_HOSTS=$(egrep -v '(^#|^\s*$)' "${SLAVES}") echo "Stopping unresponsive tablet servers (if any)..." for server in ${SLAVE_HOSTS}; do # only start if there's not one already running - $ACCUMULO_HOME/bin/stop-server.sh $server "$ACCUMULO_HOME/lib/accumulo-start.jar" tserver TERM & + "$ACCUMULO_HOME/bin/stop-server.sh" "$server" "$ACCUMULO_HOME/lib/accumulo-start.jar" tserver TERM & done sleep 10 @@ -42,8 +41,8 @@ sleep 10 echo "Stopping unresponsive tablet servers hard (if any)..." for server in ${SLAVE_HOSTS}; do # only start if there's not one already running - $ACCUMULO_HOME/bin/stop-server.sh $server "$ACCUMULO_HOME/lib/accumulo-start.jar" tserver KILL & + "$ACCUMULO_HOME/bin/stop-server.sh" "$server" "$ACCUMULO_HOME/lib/accumulo-start.jar" tserver KILL & done echo "Cleaning tablet server entries from zookeeper" -$ACCUMULO_HOME/bin/accumulo org.apache.accumulo.server.util.ZooZap -tservers +"$ACCUMULO_HOME/bin/accumulo" org.apache.accumulo.server.util.ZooZap -tservers diff --git a/assemble/bin/tool.sh b/assemble/bin/tool.sh index 376983f6e90..6830b9e6fe4 100755 --- a/assemble/bin/tool.sh +++ b/assemble/bin/tool.sh @@ -17,7 +17,7 @@ # Start: Resolve Script Directory SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink +while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located @@ -27,21 +27,21 @@ bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" . "$bin"/config.sh -if [ -z "$HADOOP_PREFIX" ] ; then +if [[ -z "$HADOOP_PREFIX" ]] ; then echo "HADOOP_PREFIX is not set. Please make sure it's set globally or in conf/accumulo-env.sh" exit 1 fi -if [ -z "$ZOOKEEPER_HOME" ] ; then +if [[ -z "$ZOOKEEPER_HOME" ]] ; then echo "ZOOKEEPER_HOME is not set. Please make sure it's set globally or in conf/accumulo-env.sh" exit 1 fi -ZOOKEEPER_CMD='ls -1 $ZOOKEEPER_HOME/zookeeper-[0-9]*[^csn].jar ' -if [ `eval $ZOOKEEPER_CMD | wc -l` != "1" ] ; then +ZOOKEEPER_CMD='ls -1 '"$ZOOKEEPER_HOME"'/zookeeper-[0-9]*[^csn].jar ' +if [[ $(eval "$ZOOKEEPER_CMD" | wc -l) != "1" ]] ; then echo "Not exactly one zookeeper jar in $ZOOKEEPER_HOME" exit 1 fi -ZOOKEEPER_LIB=$(eval $ZOOKEEPER_CMD) +ZOOKEEPER_LIB=$(eval "$ZOOKEEPER_CMD") LIB="$ACCUMULO_HOME/lib" CORE_LIB="$LIB/accumulo-core.jar" @@ -54,16 +54,16 @@ GUAVA_LIB="$LIB/guava.jar" USERJARS=" " for arg in "$@"; do - if [ "$arg" != "-libjars" -a -z "$TOOLJAR" ]; then + if [[ "$arg" != "-libjars" && -z "$TOOLJAR" ]]; then TOOLJAR="$arg" shift - elif [ "$arg" != "-libjars" -a -z "$CLASSNAME" ]; then + elif [[ "$arg" != "-libjars" && -z "$CLASSNAME" ]]; then CLASSNAME="$arg" shift - elif [ -z "$USERJARS" ]; then + elif [[ -z "$USERJARS" ]]; then USERJARS=$(echo "$arg" | tr "," " ") shift - elif [ "$arg" = "-libjars" ]; then + elif [[ "$arg" == "-libjars" ]]; then USERJARS="" shift else @@ -80,7 +80,7 @@ for jar in $USERJARS; do done export HADOOP_CLASSPATH="$H_JARS:$HADOOP_CLASSPATH" -if [ -z "$CLASSNAME" -o -z "$TOOLJAR" ]; then +if [[ -z "$CLASSNAME" || -z "$TOOLJAR" ]]; then echo "Usage: tool.sh path/to/myTool.jar my.tool.class.Name [-libjars my1.jar,my2.jar]" 1>&2 exit 1 fi @@ -89,4 +89,4 @@ fi #echo CLASSNAME=$CLASSNAME #echo HADOOP_CLASSPATH=$HADOOP_CLASSPATH #echo exec "$HADOOP_PREFIX/bin/hadoop" jar "$TOOLJAR" $CLASSNAME -libjars \"$LIB_JARS\" $ARGS -exec "$HADOOP_PREFIX/bin/hadoop" jar "$TOOLJAR" $CLASSNAME -libjars \"$LIB_JARS\" "$@" +exec "$HADOOP_PREFIX/bin/hadoop" jar "$TOOLJAR" "$CLASSNAME" -libjars \""$LIB_JARS"\" "$@" diff --git a/assemble/bin/tup.sh b/assemble/bin/tup.sh index b26def5790c..2293496b088 100755 --- a/assemble/bin/tup.sh +++ b/assemble/bin/tup.sh @@ -17,7 +17,7 @@ # Start: Resolve Script Directory SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink +while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" SOURCE="$(readlink "$SOURCE")" [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located @@ -27,20 +27,18 @@ bin="$( cd -P "$( dirname "$SOURCE" )" && pwd )" . "$bin"/config.sh -SLAVES=$ACCUMULO_CONF_DIR/slaves - echo -n "Starting tablet servers ..." count=1 -for server in `egrep -v '(^#|^\s*$)' "${SLAVES}"`; do +while IFS=$' \t\n' read -r server; do echo -n "." - ${bin}/start-server.sh $server tserver "tablet server" & + "${bin}/start-server.sh" "$server" tserver "tablet server" & let count++ - if [ $(( ${count} % 72 )) -eq 0 ] ; + if [[ $(( count % 72 )) -eq 0 ]]; then echo wait fi -done +done < <(egrep -v '^(\s*#.*|\s*)$' "$ACCUMULO_CONF_DIR/slaves") echo " done" diff --git a/assemble/conf/templates/accumulo-env.sh b/assemble/conf/templates/accumulo-env.sh index e136a3f41a8..1c04239b91c 100644 --- a/assemble/conf/templates/accumulo-env.sh +++ b/assemble/conf/templates/accumulo-env.sh @@ -27,7 +27,7 @@ ### you may want to use smaller values, especially when running everything ### on a single machine. ### -if [ -z "$HADOOP_HOME" ] +if [[ -z "$HADOOP_HOME" ]] then test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop else @@ -43,7 +43,7 @@ test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR="$HADOOP_PREFIX/etc/had test -z "$JAVA_HOME" && export JAVA_HOME=/path/to/java test -z "$ZOOKEEPER_HOME" && export ZOOKEEPER_HOME=/path/to/zookeeper test -z "$ACCUMULO_LOG_DIR" && export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs -if [ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ] +if [[ -f ${ACCUMULO_CONF_DIR}/accumulo.policy ]] then POLICY="-Djava.security.manager -Djava.security.policy=${ACCUMULO_CONF_DIR}/accumulo.policy" fi diff --git a/core/src/main/scripts/generate-thrift.sh b/core/src/main/scripts/generate-thrift.sh index 8c2a6239477..fadf272310d 100755 --- a/core/src/main/scripts/generate-thrift.sh +++ b/core/src/main/scripts/generate-thrift.sh @@ -41,7 +41,7 @@ fail() { } # Test to see if we have thrift installed -VERSION=$(thrift -version 2>/dev/null | grep -F "${REQUIRED_THRIFT_VERSION}" | wc -l | sed -e 's/^ *//' -e 's/ *$//') +VERSION=$(thrift -version 2>/dev/null | grep -c -F "${REQUIRED_THRIFT_VERSION}" | sed -e 's/^ *//' -e 's/ *$//') if [[ "${VERSION}" != '1' ]] ; then # Nope: bail echo "****************************************************" @@ -52,23 +52,23 @@ if [[ "${VERSION}" != '1' ]] ; then fi # Include thrift sources from additional modules -THRIFT_ARGS='' +THRIFT_ARGS=() for i in "${INCLUDED_MODULES[@]}"; do - if [ ${i} != '-' ]; then - test -d ${i} || fail missing required included module ${i} - THRIFT_ARGS="${THRIFT_ARGS} -I ${i}/src/main/thrift" + if [[ ${i} != '-' ]]; then + test -d "${i}" || fail missing required included module "${i}" + THRIFT_ARGS=("${THRIFT_ARGS[@]}" '-I' "${i}/src/main/thrift") fi done # Ensure output directories are created -THRIFT_ARGS="${THRIFT_ARGS} -o $BUILD_DIR" +THRIFT_ARGS=("${THRIFT_ARGS[@]}" '-o' "$BUILD_DIR") mkdir -p $BUILD_DIR rm -rf $BUILD_DIR/gen-java for f in src/main/thrift/*.thrift; do - thrift ${THRIFT_ARGS} --gen java "$f" || fail unable to generate java thrift classes - thrift ${THRIFT_ARGS} --gen py "$f" || fail unable to generate python thrift classes - thrift ${THRIFT_ARGS} --gen rb "$f" || fail unable to generate ruby thrift classes - thrift ${THRIFT_ARGS} --gen cpp "$f" || fail unable to generate cpp thrift classes + thrift "${THRIFT_ARGS[@]}" --gen java "$f" || fail unable to generate java thrift classes + thrift "${THRIFT_ARGS[@]}" --gen py "$f" || fail unable to generate python thrift classes + thrift "${THRIFT_ARGS[@]}" --gen rb "$f" || fail unable to generate ruby thrift classes + thrift "${THRIFT_ARGS[@]}" --gen cpp "$f" || fail unable to generate cpp thrift classes done # For all generated thrift code, suppress all warnings and add the LICENSE header @@ -113,7 +113,7 @@ for lang in "${LANGUAGES_TO_GENERATE[@]}"; do esac for file in "${FILE_SUFFIX[@]}"; do - for f in $(find $BUILD_DIR/gen-$lang -name "*$file"); do + for f in $(find "$BUILD_DIR/gen-$lang" -name "*$file"); do cat - "$f" > "${f}-with-license" <