-
Notifications
You must be signed in to change notification settings - Fork 2.9k
/
alluxio
executable file
·338 lines (307 loc) · 11.1 KB
/
alluxio
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
#!/usr/bin/env bash
#
# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
# (the "License"). You may not use this work except in compliance with the License, which is
# available at www.apache.org/licenses/LICENSE-2.0
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
# either express or implied, as more fully set forth in the License.
#
# See the NOTICE file distributed with this work for information regarding copyright ownership.
#
function printUsage {
echo "Usage: alluxio COMMAND [GENERIC_COMMAND_OPTIONS] [COMMAND_ARGS]"
echo
echo "COMMAND is one of:"
echo -e " format [-s] \t Format Alluxio master and all workers (if -s specified, only format if underfs is local and doesn't already exist)"
echo -e " formatWorker \t Format Alluxio worker storage"
echo -e " bootstrapConf \t Generate a config file if one doesn't exist"
echo -e " fs \t Command line input for generic filesystem user client."
echo -e " loadufs \t Load existing files in underlayer filesystem into Alluxio."
echo -e " runClass \t Run an Alluxio class with main method."
echo -e " runTest \t Run an end-to-end test on an Alluxio cluster."
echo -e " runKVTest \t Run a test of key-value store operations."
echo -e " runTests \t Run all end-to-end tests on an Alluxio cluster."
echo -e " runJournalCrashTest\t Test the Master Journal System in a crash scenario. Try 'alluxio runJournalCrashTest -help' for more help."
echo -e " \t NOTE: This command will stop the existing server and creates a new one!"
echo -e " runMesosTest \t Test the Alluxio integration with Mesos. Try 'alluxio mesosTest -help' for more help."
echo -e " \t NOTE: This command requires Mesos to be running and will stop any Alluxio servers that are currently running."
echo -e " readJournal \t Read an Alluxio journal file from stdin and write a human-readable version of it to stdout."
echo -e " killAll <WORD>\t Kill processes containing the WORD."
echo -e " copyDir <PATH>\t Copy the PATH to all worker nodes."
echo -e " clearCache \t Clear OS buffer cache of the machine."
echo -e " thriftGen \t Generate all thrift code."
echo -e " protoGen \t Generate all protocol buffer code."
echo -e " version \t Print Alluxio version and exit."
echo -e " validateConf \t Validate Alluxio conf and exit."
echo
echo "GENERIC_COMMAND_OPTIONS supports:"
echo -e " -D<property=value>\t Use a value for a given Alluxio property"
echo
# TODO(binfan): Fix help function for alluxio script
echo "Commands print help when invoked without parameters."
}
function bootstrapConf {
local usage="Usage: $0 bootstrapConf <alluxio_master_hostname>"
if [[ $# -lt 1 ]]; then
echo ${usage} >&2
exit 1
fi
local master=${1}
ALLUXIO_CONF_DIR="${BIN}/../conf"
local dst="${ALLUXIO_CONF_DIR}/alluxio-env.sh"
if [[ -e ${dst} ]]; then
echo "${dst} already exists"
return
fi
if [[ $(uname -s) == Darwin ]]; then
# Assuming Mac OS X
TOTAL_MEM=$(sysctl hw.memsize | cut -d ' ' -f2)
TOTAL_MEM=$[TOTAL_MEM / 1024 / 1024]
TOTAL_MEM=$[TOTAL_MEM * 2 / 3]
RAMDISK_FOLDER="/Volumes/ramdisk"
else
# Assuming Linux
TOTAL_MEM=$(awk '/MemTotal/{print $2}' /proc/meminfo)
TOTAL_MEM=$[TOTAL_MEM / 1024 * 2 / 3]
RAMDISK_FOLDER="/mnt/ramdisk"
fi
# Set the master hostname
ALLUXIO_MASTER_HOSTNAME=${master}
# Set the worker memory size
ALLUXIO_WORKER_MEMORY_SIZE="${TOTAL_MEM}MB"
# Set the ramdisk folder
ALLUXIO_RAM_FOLDER="${RAMDISK_FOLDER}"
cat > ${dst} <<EOF
#!/usr/bin/env bash
#
# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
# (the "License"). You may not use this work except in compliance with the License, which is
# available at www.apache.org/licenses/LICENSE-2.0
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
# either express or implied, as more fully set forth in the License.
#
# See the NOTICE file distributed with this work for information regarding copyright ownership.
#
# This file is sourced to launch Alluxio servers or use Alluxio shell commands
# This file was generated by command "bin/alluxio bootstrapConf ${master} ${ufs}"
# For a list of supported environment variables, see "conf/alluxio-env.sh.template"
ALLUXIO_MASTER_HOSTNAME=\${ALLUXIO_MASTER_HOSTNAME:-"${ALLUXIO_MASTER_HOSTNAME}"}
ALLUXIO_WORKER_MEMORY_SIZE=\${ALLUXIO_WORKER_MEMORY_SIZE:-"${ALLUXIO_WORKER_MEMORY_SIZE}"}
ALLUXIO_RAM_FOLDER=\${ALLUXIO_RAM_FOLDER:-"${ALLUXIO_RAM_FOLDER}"}
EOF
echo "${dst} is created."
}
function killAll {
if [[ $# -ne 1 ]]; then
echo "Usage: alluxio killAll <WORD>"
exit
fi
keyword=$1
count=0
for pid in `ps -Aww -o pid,command | grep -i "[j]ava" | grep ${keyword} | awk '{print $1}'`; do
kill -15 ${pid} > /dev/null 2>&1
local cnt=30
while kill -0 ${pid} > /dev/null 2>&1; do
if [[ ${cnt} -gt 1 ]]; then
# still not dead, wait
cnt=`expr ${cnt} - 1`
sleep 1
else
# waited long enough, kill the process
echo "Process did not complete after 30 seconds, killing."
kill -9 ${pid} 2> /dev/null
fi
done
count=`expr ${count} + 1`
done
echo "Killed ${count} processes on `hostname`"
}
function copyDir {
if [[ $# -ne 1 ]]; then
echo "Usage: alluxio copyDir <path>" >&2
exit 1
fi
WORKERS=`cat ${ALLUXIO_CONF_DIR}/workers | grep -v '^#'`
DIR=`readlink -f "$1"`
DIR=`echo ${DIR}|sed 's@/$@@'`
DEST=`dirname ${DIR}`
SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=5"
echo "RSYNC'ing ${DIR} to workers..."
for worker in ${WORKERS}; do
echo ${worker}
rsync -e "ssh ${SSH_OPTS}" -az ${DIR} ${worker}:${DEST} & sleep 0.05
done
wait
}
function runJavaClass {
CLASS_ARGS=""
for arg in "$@"; do
shift
case "${arg}" in
-D*) ALLUXIO_SHELL_JAVA_OPTS+=" ${arg}" ;;
*) CLASS_ARGS+=" ${arg}"
esac
done
${JAVA} -cp ${CLASSPATH} ${ALLUXIO_USER_JAVA_OPTS} ${ALLUXIO_SHELL_JAVA_OPTS} ${CLASS} ${PARAMETER} ${CLASS_ARGS}
}
function main {
LAUNCHER=
# If debugging is enabled propagate that through to sub-shells
if [[ $- == *x* ]]; then
LAUNCHER="bash -x"
fi
BIN=$(cd "$( dirname "$0" )"; pwd)
if [[ $# == 0 ]]; then
printUsage
exit 1
fi
COMMAND=$1
shift
# bootstrap to create conf/alluxio-env.sh. This commands needs to be processed
# before sourcing libexec/alluxio-conf.sh which requires conf/alluxio-env.sh
if [[ "${COMMAND}" == "bootstrapConf" || "${COMMAND}" == "bootstrap-conf" ]]; then
bootstrapConf "$@"
exit 0
fi
DEFAULT_LIBEXEC_DIR="${BIN}"/../libexec
ALLUXIO_LIBEXEC_DIR=${ALLUXIO_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. ${ALLUXIO_LIBEXEC_DIR}/alluxio-config.sh
PARAMETER=""
ALLUXIO_SHELL_JAVA_OPTS=""
case ${COMMAND} in
"format")
if [[ $# -eq 1 ]]; then
if [[ $1 == "-s" ]]; then
if [[ -e ${ALLUXIO_UNDERFS_ADDRESS} ]]; then
# if ufs is local filesystem and already exists
exit 0
else
declare -a schemes=(hdfs s3 s3a s3n glusterfs swift oss gcs)
for scheme in ${schemes[@]}; do
if [[ ${ALLUXIO_UNDERFS_ADDRESS} == ${scheme}://* ]]; then
# if ufs is not local filesystem, don't format
exit 0
fi
done
shift # remove -s param
fi
else
echo "{Usage} $0 format [-s]"
exit 2
fi
elif [[ $# -gt 1 ]]; then
echo "${Usage} $0 format [-s]"
exit 2
fi
${LAUNCHER} ${BIN}/alluxio-workers.sh ${BIN}/alluxio formatWorker
echo "Formatting Alluxio Master @ ${ALLUXIO_MASTER_HOSTNAME}"
CLASS="alluxio.cli.Format"
PARAMETER="master"
runJavaClass "$@"
;;
"formatWorker")
echo "Formatting Alluxio Worker @ `hostname -f`"
CLASS="alluxio.cli.Format"
PARAMETER="worker"
runJavaClass "$@"
;;
"fs")
CLASS="alluxio.cli.AlluxioShell"
runJavaClass "$@"
;;
"loadufs")
echo "The \"alluxio loadufs <AlluxioURI> <UfsURI>\" command is deprecated since version 1.1." >&2
echo "Use the \"alluxio fs mount <AlluxioURI> <UfsURI>\" command followed by the \"alluxio fs ls -R <AlluxioURI>\" command instead." >&2
exit 1
;;
"runClass")
CLASS=$1
shift
runJavaClass "$@"
;;
"runTest")
CLASS="alluxio.cli.TestRunner"
PARAMETER="alluxio://${ALLUXIO_MASTER_HOSTNAME}:19998"
ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console"
runJavaClass "$@"
;;
"runKVTest")
CLASS="alluxio.examples.keyvalue.KeyValueStoreOperations"
PARAMETER="/default_tests_files/KeyValueStoreOperations"
ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console"
runJavaClass "$@"
;;
"runTests")
CLASS="alluxio.cli.TestRunner"
PARAMETER="alluxio://${ALLUXIO_MASTER_HOSTNAME}:19998"
ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console"
runJavaClass "$@"
;;
"runMiniBenchmark")
CLASS="alluxio.cli.MiniBenchmark"
PARAMETER="alluxio://${ALLUXIO_MASTER_HOSTNAME}:19998"
ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console"
runJavaClass "$@"
;;
"runJournalCrashTest")
CLASS="alluxio.cli.JournalCrashTest"
ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console"
runJavaClass "$@"
;;
"runMesosTest")
CLASS="alluxio.cli.AlluxioFrameworkIntegrationTest"
ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console"
runJavaClass "$@"
;;
"readJournal")
CLASS="alluxio.master.journal.JournalTool"
ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console"
runJavaClass "$@"
;;
"killAll")
killAll "$@"
;;
"copyDir")
copyDir "$@"
;;
"thriftGen")
declare -a thriftSrcPaths=($(find "${BIN}/.." -name "*.thrift" -print0 | xargs -0 -n1 dirname | sort | uniq))
for thriftSrcPath in ${thriftSrcPaths[@]}; do
rm -rf "${thriftSrcPath}/../main/java/alluxio/thrift"
for srcFile in $(ls "${thriftSrcPath}"); do
thrift -I "${BIN}/../core/common/src/thrift" --gen java:private-members,generated_annotations=undated -out "${thriftSrcPath}/../main/java/." "${thriftSrcPath}/${srcFile}"
done
done
;;
"protoGen")
rm -rf ${BIN}/../core/server/src/main/java/alluxio/proto
for src_file in $(find ${BIN}/../core/server/src/proto -name '*.proto' -type f); do
protoc --java_out=${BIN}/../core/server/src/main/java --proto_path=`dirname ${src_file}` ${src_file}
done
rm -rf ${BIN}/../core/common/src/main/java/alluxio/proto
for src_file in $(find ${BIN}/../core/common/src/proto -name '*.proto' -type f); do
protoc --java_out=${BIN}/../core/common/src/main/java --proto_path=`dirname ${src_file}` ${src_file}
done
;;
"clearCache")
sync; echo 3 > /proc/sys/vm/drop_caches ;
;;
"version")
CLASS="alluxio.cli.Version"
runJavaClass "$@"
;;
"validateConf")
CLASS="alluxio.cli.ValidateConf"
ALLUXIO_SHELL_JAVA_OPTS+=" -Dalluxio.logger.type=Console"
runJavaClass "$@"
;;
*)
echo "Unsupported command ${COMMAND}" >&2
printUsage
exit 1
;;
esac
}
main "$@"