/
setenv.sh
143 lines (116 loc) · 2.99 KB
/
setenv.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
#
# you must set EMR_HOME to point to the root directory of your 'elastic-mapreduce-ruby' install
#
export PATH=$EMR_HOME:$PATH
[ -z "$EMR_CRED_JSON" ] && EMR_CRED_JSON=$EMR_HOME/credentials.json
if [ ! -f $EMR_CRED_JSON ];then
echo "Credentias at $EMR_CRED_JSON do not exist!"
else
echo "Using EMR credentials: $EMR_CRED_JSON"
fi
# EMR helpers
export EMR_SSH_KEY=`cat $EMR_CRED_JSON | grep '"key-pair-file"' | cut -d':' -f2 | sed -n 's|.*"\([^"]*\)".*|\1|p'`
export EMR_SSH_KEY_NAME=`cat $EMR_CRED_JSON | grep '"key-pair"' | cut -d':' -f2 | sed -n 's|.*"\([^"]*\)".*|\1|p'`
export EMR_SSH_OPTS="-i "$EMR_SSH_KEY" -o StrictHostKeyChecking=no -o ServerAliveInterval=30"
export ELASTIC_MAPREDUCE_CREDENTIALS=$EMR_CRED_JSON
function emr {
RESULT=`elastic-mapreduce $*`
ID=`echo "$RESULT" | head -1 | sed -n 's|^Cr.*\(j-[^ ]*\)$|\1|p'`
[ -n "$ID" ] && export EMR_FLOW_ID="$ID"
echo "$RESULT"
}
function emrset {
if [ -z "$1" ]; then
echo $EMR_FLOW_ID
else
export EMR_FLOW_ID=$1
fi
}
function flowid {
if [ -z "$EMR_FLOW_ID" ]; then
echo "$1"
else
echo "$EMR_FLOW_ID"
fi
}
function emrhost {
if [[ $1 =~ ^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$ ]]; then
echo $1
return
fi
FLOW_ID=`flowid $1`
unset H
while [ -z "$H" ]; do
H=`emr -j $FLOW_ID --describe | grep MasterPublicDnsName | sed -n 's|.*"\([^"]*\)".*|\1|p'`
sleep 5
done
echo $H
}
function emrscreen {
HOST=`emrhost $1`
ssh $EMR_SSH_OPTS -t "hadoop@$HOST" 'screen -s -$SHELL -D -R'
}
function emrtail {
if [ -z "$1" ]; then
echo "Must provide step number to tail!"
HOST=`emrhost $HH`
ssh $EMR_SSH_OPTS -t "hadoop@$HOST" "ls -1 /mnt/var/log/hadoop/steps/"
return
fi
if [ $# == 2 ]; then
HH=$1
STEP=$2
else
HH=""
STEP=$1
fi
HOST=`emrhost $HH`
ssh $EMR_SSH_OPTS -t "hadoop@$HOST" "tail -100f /mnt/var/log/hadoop/steps/$STEP/syslog"
}
function emrlogin {
HOST=`emrhost $1`
ssh $EMR_SSH_OPTS "hadoop@$HOST"
}
function emrproxy {
HOST=`emrhost $1`
echo "JobTracker: http://$HOST:9100"
echo "NameNode : http://$HOST:9101"
ssh $EMR_SSH_OPTS -D 6666 -N "hadoop@$HOST"
}
function emrlist {
local opts=""
if [[ $* != *-v* ]]; then
opts="$opts --no-steps --active"
fi
emr --list $opts
}
function emrstat {
FLOW_ID=`flowid $1`
emr -j $FLOW_ID --describe | grep 'LastStateChangeReason' | sort -r | head -1 | cut -d":" -f2 | sed -n 's|^ "\([^\"]*\)".*|\1|p'
}
function emrterminate {
if [ "$1" == -f ]; then f=1; shift; fi
FLOW_ID=`flowid $1`
[ -n "$f" ] && emr -j $FLOW_ID --set-termination-protection false
emr -j $FLOW_ID --terminate
export EMR_FLOW_ID=""
}
function emrscp {
HOST=`emrhost`
scp $EMR_SSH_OPTS $1 "hadoop@$HOST:"
}
function emrconf {
if [ -z "$1" ]; then
echo "Must provide target directory to place files!"
return
fi
if [ $# == 2 ]; then
HH=$1
CONFPATH=$2
else
HH=""
CONFPATH=$1
fi
HOST=`emrhost $HH`
scp $EMR_SSH_OPTS "hadoop@$HOST:conf/*-site.xml" $CONFPATH/
}