Permalink
executable file 295 lines (253 sloc) 9.66 KB
#!/usr/bin/env /bin/bash
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##
# This script runs core flume commands. It sets up the proper jar
# library paths and defaults to a java class name.
#
# TODO (jon) This assumes CLASSPATH has $JAVA_HOME/lib/tools.jar in it
# (jetty requires this to compile jsps. In future will precompile
# jsps to make this requirement go away.)
function usage {
echo "usage: flume command [args...]"
echo "commands include: "
echo " dump Takes a specified source and dumps to console"
echo " source Takes a specified source and dumps to console"
echo " node Start a Flume node/agent (with watchdog)"
echo " master Start a Flume Master server (with watchdog)"
echo " version Dump flume build version information "
echo " node_nowatch Start a flume node/agent (no watchdog)"
echo " master_nowatch Start a Flume Master server (no watchdog)"
echo " class <class> Run specified fully qualified class using Flume environment (no watchdog)"
echo " ex: flume com.cloudera.flume.agent.FlumeNode "
echo " classpath Dump the classpath used by the java executables"
echo " shell Start the flume shell"
echo " killmaster Kill a running master"
echo " dumplog Takes a specified WAL/DFO log file and dumps to console"
echo " sink Start a one-shot flume node with console source and specified sink"
exit -1
}
CMDPATH=`dirname $0`
# This is to make CMDPATH correct if you go to the bin directory
# and run ./flume <command>.
pushd $CMDPATH >/dev/null
CMDPATH=`pwd`
popd >/dev/null
# name of path
CMDPATH=`dirname $CMDPATH`
FLUME=$CMDPATH/bin/flume
MASTERPID=${FLUME_PID_DIR:="/tmp"}/flumemaster.pid
CMD="$1"
if [ -z "$CMD" ]; then
usage
fi
cygwin=false
pathsep=":"
case "`uname`" in
CYGWIN*)
cygwin=true
pathsep=";"
;;
esac
function append_jars_onto_classpath() {
local JARS
JARS=`find $1/*.jar 2> /dev/null || true`
for i in $JARS; do
if [ -n "$CLASSPATH" ]; then
CLASSPATH=${CLASSPATH}${pathsep}${i}
else
CLASSPATH=${i}
fi
done
}
# name of script
BINPATH=`dirname $0`
if [ -f "${BINPATH}/flume-env.sh" ]; then
source "$BINPATH/flume-env.sh"
fi
if [ -z "$FLUME_HOME" ]; then
export FLUME_HOME="$CMDPATH"
fi
JOPTS="-Dflume.log.dir=${FLUME_LOG_DIR:-${FLUME_HOME}/logs} "
JOPTS="$JOPTS -Dflume.log.file=${FLUME_LOGFILE:-flume.log} "
JOPTS="$JOPTS -Dflume.root.logger=${FLUME_ROOT_LOGGER:-INFO,console} "
JOPTS="$JOPTS -Dzookeeper.root.logger=${ZOOKEEPER_ROOT_LOGGER:-ERROR,console} "
JOPTS="$JOPTS -Dwatchdog.root.logger=${WATCHDOG_ROOT_LOGGER:-INFO,console} "
if [ "$cygwin" == false ] ; then
# unix and "unix-like" setup
# pick user override, or check for dev env, or check for install
if [ -n "$FLUME_CONF_DIR" ]; then
true
elif [ -e "./conf/flume-conf.xml" ]; then
export FLUME_CONF_DIR="./conf"
elif [ -e "/etc/flume/conf/flume-conf.xml" ]; then
export FLUME_CONF_DIR="/etc/flume/conf"
else
echo "FLUME_CONF_DIR cannot be determined, please set explicitly"
exit -1
fi
# conf dir first in the class path in order to ensure we get flume log4j.properties
CLASSPATH="$FLUME_CONF_DIR"
if [ -n "$FLUME_CLASSPATH" ]; then
CLASSPATH="${CLASSPATH}:${FLUME_CLASSPATH}"
fi
# put hadoop conf dir in classpath to include Hadoop
# core-site.xml/hdfs-site.xml
if [ -n "${HADOOP_CONF_DIR}" ]; then
CLASSPATH="${CLASSPATH}:${HADOOP_CONF_DIR}"
elif [ -n "${HADOOP_HOME}" ] ; then
CLASSPATH="${CLASSPATH}:${HADOOP_HOME}/conf"
elif [ -e "/usr/lib/hadoop/conf" ] ; then
# if neither is present see if the CDH dir exists
CLASSPATH="${CLASSPATH}:/usr/lib/hadoop-0.20/conf";
fi # otherwise give up
# try to load the hadoop core jars
HADOOP_CORE_FOUND=false
while true; do
if [ -n "$HADOOP_HOME" ]; then
HADCOREJARS=`find ${HADOOP_HOME}/hadoop-core*.jar || find ${HADOOP_HOME}/lib/hadoop-core*.jar || true`
if [ -n "$HADCOREJARS" ]; then
HADOOP_CORE_FOUND=true
CLASSPATH="$CLASSPATH:${HADCOREJARS}"
break;
fi
fi
HADCOREJARS=`find ./lib/hadoop-core*.jar 2> /dev/null || true`
if [ -n "$HADCOREJARS" ]; then
# if this is the dev environment then hadoop jar will
# get added as part of ./lib (below)
break
fi
# core jars may be missing, we'll check for this below
break
done
JOPTS="$JOPTS -Djava.library.path=$CMDPATH/lib:$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATH"
append_jars_onto_classpath "$CMDPATH/lib"
append_jars_onto_classpath "$CMDPATH/libtest"
CLASSPATH="${CLASSPATH}:$JAVA_HOME/lib/tools.jar"
if [ -z "$ZOOKEEPER_HOME" ]; then
export ZOOKEEPER_HOME="/usr/lib/zookeeper"
fi
append_jars_onto_classpath "${ZOOKEEPER_HOME}"
CLASSPATH="$CLASSPATH:$CMDPATH/build/classes"
append_jars_onto_classpath "$CMDPATH/build"
append_jars_onto_classpath "${FLUME_HOME}"
else
# windows with cygwin
JOPTS="$JOPTS -Djava.library.path=`cygpath -d $CMDPATH/lib` "
CLASSPATH="$JAVA_HOME/lib/tools.jar"
append_jars_onto_classpath "$CMDPATH/lib"
append_jars_onto_classpath "$CMDPATH/libtest"
fi
export CLASSPATH
# if we found hadoop core already then no need to test
# however if we're not certain then check the classpath
if [ $HADOOP_CORE_FOUND == false ]; then
LENCP=${#CLASSPATH}
# replace hadoop-core*.jar with 0 len string
HADTEST=${CLASSPATH/hadoop-core*.jar/}
HADTESTLEN=${#HADTEST}
# lengths should be different if jars are in classpath
if [ $LENCP -eq $HADTESTLEN ]; then
echo "HADOOP_HOME is unset, hadoop jars may not be added to classpath"
fi
fi
if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then
. "${HADOOP_CONF_DIR}/hadoop-env.sh"
fi
shift
# pass properties to allow java program to drop a pid file. java must
# now be exec'ed so the pid remains the same as the script. In the
# mastercase, we actually only drop the pid of the watchdog. The
# watchdog kills the watched child process if killed cleanly.
WATCHDOG_OPTS="-Dpid=$$ -Dpidfile="
WATCHDOG_CLASS=com.cloudera.flume.watchdog.FlumeWatchdog
MASTER_WATCHDOG="java ${JOPTS} -Dpid=$$ -Dpidfile=$MASTERPID $WATCHDOG_CLASS"
MASTERI_WATCHDOG="java ${JOPTS} -Dpid=$$ -Dpidfile=$MASTERPID $WATCHDOG_CLASS"
NODE_WATCHDOG="java ${JOPTS} -Dpid=$$ -Dpidfile=${FLUME_PID_DIR}/flumenode-$$.pid $WATCHDOG_CLASS"
NODEI_WATCHDOG="java ${JOPTS} -Dfwdstdin=true -Dpid=$$ -Dpidfile=${FLUME_PID_DIR}/flumenode-$$.pid $WATCHDOG_CLASS"
if [ -n "$FLUME_VERBOSE" ]; then
if [ -n "$FLUME_VERBOSE_JAVA" ]; then
JOPTS="$JOPTS -verbose "
fi
echo "$CLASSPATH"
set -x
fi
if [ "$CMD" = "-h" ] ; then
usage
elif [ "$CMD" = "node" ] ; then
# there can be multiple nodes.
exec $NODE_WATCHDOG java $JOPTS $UOPTS com.cloudera.flume.agent.FlumeNode "$@"
elif [ "$CMD" = "nodei" ] ; then
# there can be multiple nodes.
exec $NODEI_WATCHDOG java $JOPTS $UOPTS com.cloudera.flume.agent.FlumeNode "$@"
elif [ "$CMD" = "master" ] ; then
# only allow one master
if [ -f $MASTERPID ] ; then
PID=`cat $MASTERPID`
echo "Master already running, pid=$PID"
exit -1
fi
exec $MASTER_WATCHDOG java $JOPTS $UOPTS com.cloudera.flume.master.FlumeMaster "$@"
elif [ "$CMD" = "node_nowatch" ] ; then
exec java $JOPTS $UOPTS com.cloudera.flume.agent.FlumeNode "$@"
elif [ "$CMD" = "master_nowatch" ] ; then
exec java $JOPTS $UOPTS com.cloudera.flume.master.FlumeMaster "$@"
elif [ "$CMD" = "watchdog" ] ; then
exec java $UOPTS $WATCHDOG_CLASS "$@"
elif [ "$CMD" = "version" ] ; then
exec java $JOPTS $UOPTS com.cloudera.flume.VersionInfo
elif [ "$CMD" = "unit" ] ; then
exec $FLUME junit.textui.TestRunner "$@"
elif [ "$CMD" = "unit4" ] ; then
exec $FLUME org.junit.runner.JUnitCore "$@"
elif [ "$CMD" = "dump" -o "$CMD" = "source" ] ; then
# this has no error checking. be careful!
if [ -z "$2" ] ; then
CONSOLE="console"
else
CONSOLE="console(\"$2\")"
fi
exec java $JOPTS $UOPTS com.cloudera.flume.agent.FlumeNode \
-1 -s -n dump -c "dump: $1 | $CONSOLE; "
elif [ "$CMD" = "dumplog" ] ; then
# this has no error checking. be careful!
exec java $JOPTS $UOPTS com.cloudera.flume.agent.FlumeNode \
-1 -s -n dump -c "dump: seqfile(\"$1\") | console(\"avrojson\"); "
elif [ "$CMD" = "sink" ]; then
exec java $JOPTS $UOPTS com.cloudera.flume.agent.FlumeNode \
-1 -s -n dump -c "dump: console | $1; "
elif [ "$CMD" = "shell" ] ; then
exec java $JOPTS $UOPTS com.cloudera.flume.util.FlumeShell "$@"
elif [ "$CMD" = "killmaster" ] ; then
if [ -f $MASTERPID ] ; then
PID=`cat $MASTERPID`
echo "Killing FlumeMaster (pid=$PID)"
if kill $PID ; then
echo "FlumeMaster stopped"
else
echo "FlumeMaster (pid=$PID) could not be stopped"
fi
rm -f $MASTERPID
fi
elif [ "$CMD" = "class" ]; then
# Just do a java class with the environment setup
exec java $JOPTS $UOPTS "$@"
elif [ "$CMD" = "classpath" ]; then
echo $CLASSPATH
else
usage
fi