-
Notifications
You must be signed in to change notification settings - Fork 214
/
flume
executable file
·294 lines (250 loc) · 9.25 KB
/
flume
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
#!/usr/bin/env /bin/bash
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##
# This script runs core flume commands. It sets up the proper jar
# library paths and defaults to a java class name.
#
# TODO (jon) This assumes CLASSPATH has $JAVA_HOME/lib/tools.jar in it
# (jetty requires this to compile jsps. In future will precompile
# jsps to make this requirement go away.)
function usage {
echo "usage: flume command [args...]"
echo "commands include: "
echo " dump Takes a specified source and dumps to console"
echo " node Start a Flume node/agent (with watchdog)"
echo " master Start a Flume Master server (with watchdog)"
echo " version Dump flume build version information "
echo " node_nowatch Start a flume node/agent (no watchdog)"
echo " master_nowatch Start a Flume Master server (no watchdog)"
echo " class <class> Run specified fully qualified class using Flume environment (no watchdog)"
echo " ex: flume com.cloudera.flume.agent.FlumeNode "
echo " classpath Dump the classpath used by the java executables"
echo " shell Start the flume shell"
echo " killmaster Kill a running master"
exit -1
}
CMDPATH=`dirname $0`
# This is to make CMDPATH correct if you go to the bin directory
# and run ./flume <command>.
pushd $CMDPATH >/dev/null
CMDPATH=`pwd`
popd >/dev/null
# name of path
CMDPATH=`dirname $CMDPATH`
FLUME=$CMDPATH/bin/flume
MASTERPID=${FLUME_PID_DIR:="/tmp"}/flumemaster.pid
CMD="$1"
if [ -z "$CMD" ]; then
usage
fi
cygwin=false
pathsep=":"
case "`uname`" in
CYGWIN*)
cygwin=true
pathsep=";"
;;
esac
function append_jars_onto_classpath() {
local JARS
JARS=`find $1/*.jar 2> /dev/null || true`
for i in $JARS; do
if [ -n "$CLASSPATH" ]; then
CLASSPATH+=${pathsep}${i}
else
CLASSPATH=${i}
fi
done
}
# name of script
BINPATH=`dirname $0`
if [ -f "${BINPATH}/flume-env.sh" ]; then
source "$BINPATH/flume-env.sh"
fi
if [ -z "$FLUME_HOME" ]; then
export FLUME_HOME="$CMDPATH"
fi
JOPTS="-Dflume.log.dir=${FLUME_LOG_DIR:-${FLUME_HOME}/logs} "
JOPTS+="-Dflume.log.file=${FLUME_LOGFILE:-flume.log} "
JOPTS+="-Dflume.root.logger=${FLUME_ROOT_LOGGER:-INFO,console} "
JOPTS+="-Dzookeeper.root.logger=${ZOOKEEPER_ROOT_LOGGER:-ERROR,console} "
JOPTS+="-Dwatchdog.root.logger=${WATCHDOG_ROOT_LOGGER:-INFO,console} "
if [ "$cygwin" == false ] ; then
# unix and "unix-like" setup
# pick user override, or check for dev env, or check for install
if [ -n "$FLUME_CONF_DIR" ]; then
true
elif [ -e "./conf/flume-conf.xml" ]; then
export FLUME_CONF_DIR="./conf"
elif [ -e "/etc/flume/conf/flume-conf.xml" ]; then
export FLUME_CONF_DIR="/etc/flume/conf"
else
echo "FLUME_CONF_DIR cannot be determined, please set explicitly"
exit -1
fi
# conf dir first in the class path in order to ensure we get flume log4j.properties
CLASSPATH="$FLUME_CONF_DIR"
if [ -n "$FLUME_CLASSPATH" ]; then
CLASSPATH+=":${FLUME_CLASSPATH}"
fi
# put hadoop conf dir in classpath to include Hadoop
# core-site.xml/hdfs-site.xml
if [ -n "${HADOOP_CONF_DIR}" ]; then
CLASSPATH+=":${HADOOP_CONF_DIR}"
elif [ -n "${HADOOP_HOME}" ] ; then
CLASSPATH+=":${HADOOP_HOME}/conf"
elif [ -e "/usr/lib/hadoop/conf" ] ; then
# if neither is present see if the CDH dir exists
CLASSPATH+=":/usr/lib/hadoop-0.20/conf";
fi # otherwise give up
# try to load the hadoop core jars
HADOOP_CORE_FOUND=false
while true; do
if [ -n "$HADOOP_HOME" ]; then
HADOOP_CORE_FOUND=true
CLASSPATH+=":${HADOOP_HOME}/hadoop-*-core.jar:${HADOOP_HOME}/lib/hadoop-*-core.jar"
break
fi
HADCOREJARS=`find /usr/lib/hadoop/hadoop-*core.jar 2> /dev/null || true`
if [ -n "$HADCOREJARS" ]; then
HADOOP_CORE_FOUND=true
export CDH_CLASSPATH="/usr/lib/hadoop"
CLASSPATH+=":${CDH_CLASSPATH}/*"
break
fi
HADCOREJARS=`find ./lib/hadoop-*core.jar 2> /dev/null || true`
if [ -n "$HADCOREJARS" ]; then
# if this is the dev environment then hadoop jar will
# get added as part of ./lib (below)
break
fi
# core jars may be missing, we'll check for this below
break
done
JOPTS+="-Djava.library.path=$CMDPATH/lib "
append_jars_onto_classpath "$CMDPATH/lib"
append_jars_onto_classpath "$CMDPATH/libtest"
CLASSPATH+=":$JAVA_HOME/lib/tools.jar"
if [ "$FLUME_DEVMODE" == true ] ; then
# allow Jetty to compile servlets in dev mode
append_jars_onto_classpath "$CMDPATH/libbuild"
fi
if [ -z "$ZOOKEEPER_HOME" ]; then
export ZOOKEEPER_HOME="/usr/lib/zookeeper"
fi
append_jars_onto_classpath "${ZOOKEEPER_HOME}"
CLASSPATH+=":$CMDPATH/build/classes"
append_jars_onto_classpath "$CMDPATH/build"
append_jars_onto_classpath "${FLUME_HOME}"
else
# windows with cygwin
JOPTS+="-Djava.library.path=`cygpath -d $CMDPATH/lib` "
CLASSPATH="$JAVA_HOME/lib/tools.jar"
append_jars_onto_classpath "$CMDPATH/lib"
append_jars_onto_classpath "$CMDPATH/libtest"
if [ "$FLUME_DEVMODE" == true ] ; then
# allow Jetty to compile servlets in dev mode
append_jars_onto_classpath "$CMDPATH/libbuild"
fi
fi
export CLASSPATH
# if we found hadoop core already then no need to test
# however if we're not certain then check the classpath
if [ $HADOOP_CORE_FOUND == false ]; then
LENCP=${#CLASSPATH}
# replace hadoop-*core.jar with 0 len string
HADTEST=${CLASSPATH/hadoop-*core.jar/}
HADTESTLEN=${#HADTEST}
# lengths should be different if jars are in classpath
if [ $LENCP -eq $HADTESTLEN ]; then
echo "HADOOP_HOME is unset, hadoop jars may not be added to classpath"
fi
fi
if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then
. "${HADOOP_CONF_DIR}/hadoop-env.sh"
fi
shift
# pass properties to allow java program to drop a pid file. java must
# now be exec'ed so the pid remains the same as the script. In the
# mastercase, we actually only drop the pid of the watchdog. The
# watchdog kills the watched child process if killed cleanly.
WATCHDOG_OPTS="-Dpid=$$ -Dpidfile="
WATCHDOG_CLASS=com.cloudera.flume.watchdog.FlumeWatchdog
MASTER_WATCHDOG="java ${JOPTS} -Dpid=$$ -Dpidfile=$MASTERPID $WATCHDOG_CLASS"
MASTERI_WATCHDOG="java ${JOPTS} -Dpid=$$ -Dpidfile=$MASTERPID $WATCHDOG_CLASS"
NODE_WATCHDOG="java ${JOPTS} -Dpid=$$ -Dpidfile=${FLUME_PID_DIR}/flumenode-$$.pid $WATCHDOG_CLASS"
NODEI_WATCHDOG="java ${JOPTS} -Dfwdstdin=true -Dpid=$$ -Dpidfile=${FLUME_PID_DIR}/flumenode-$$.pid $WATCHDOG_CLASS"
if [ -n "$FLUME_VERBOSE" ]; then
if [ -n "$FLUME_VERBOSE_JAVA" ]; then
JOPTS+="-verbose "
fi
echo "$CLASSPATH"
set -x
fi
if [ "$CMD" = "-h" ] ; then
usage
elif [ "$CMD" = "node" ] ; then
# there can be multiple nodes.
exec $NODE_WATCHDOG java $JOPTS $UOPTS com.cloudera.flume.agent.FlumeNode "$@"
elif [ "$CMD" = "nodei" ] ; then
# there can be multiple nodes.
exec $NODEI_WATCHDOG java $JOPTS $UOPTS com.cloudera.flume.agent.FlumeNode "$@"
elif [ "$CMD" = "master" ] ; then
# only allow one master
if [ -f $MASTERPID ] ; then
PID=`cat $MASTERPID`
echo "Master already running, pid=$PID"
exit -1
fi
exec $MASTER_WATCHDOG java $JOPTS $UOPTS com.cloudera.flume.master.FlumeMaster "$@"
elif [ "$CMD" = "node_nowatch" ] ; then
exec java $JOPTS $UOPTS com.cloudera.flume.agent.FlumeNode "$@"
elif [ "$CMD" = "master_nowatch" ] ; then
exec java $JOPTS $UOPTS com.cloudera.flume.master.FlumeMaster "$@"
elif [ "$CMD" = "watchdog" ] ; then
exec java $UOPTS $WATCHDOG_CLASS "$@"
elif [ "$CMD" = "version" ] ; then
exec java $JOPTS $UOPTS com.cloudera.flume.VersionInfo
elif [ "$CMD" = "unit" ] ; then
exec $FLUME junit.textui.TestRunner "$@"
elif [ "$CMD" = "unit4" ] ; then
exec $FLUME org.junit.runner.JUnitCore "$@"
elif [ "$CMD" = "dump" ] ; then
# this has no error checking. be careful!
exec java $JOPTS $UOPTS com.cloudera.flume.agent.FlumeNode \
-1 -s -r -n dump -c "dump: $1 | console; "
elif [ "$CMD" = "shell" ] ; then
exec java $JOPTS $UOPTS com.cloudera.flume.util.FlumeShell "$@"
elif [ "$CMD" = "killmaster" ] ; then
if [ -f $MASTERPID ] ; then
PID=`cat $MASTERPID`
echo "Killing FlumeMaster (pid=$PID)"
if kill $PID ; then
echo "FlumeMaster stopped"
else
echo "FlumeMaster (pid=$PID) could not be stopped"
fi
rm -f $MASTERPID
fi
elif [ "$CMD" = "class" ]; then
# Just do a java class with the environment setup
exec java $JOPTS $UOPTS "$@"
elif [ "$CMD" = "classpath" ]; then
echo $CLASSPATH
else
usage
fi