-
Notifications
You must be signed in to change notification settings - Fork 3
/
deployer.sh
executable file
·276 lines (233 loc) · 7.43 KB
/
deployer.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
#!/bin/bash
#
# Implements a deployer script through SSH. The user should have ssh access
# to the hosts which workers will be deployed to. It is required to have
# access without passwords and prompts.
#
# The script will perform the following tasks to deploy worker:
# 1. Create a configuration file for each host by copying on a base
# configuration and altering hostname and port options
# 2. Create a .spiderpig folder in the user home folder of each host
# 3. Create a subfolder for the worker
# 4. Copy the worker package to that .spiderpig subfolder
# 5. SSH to the host and execute the start comand
#
# After deployment, steps 1-4 are ignored and only a command is executed on
# step 4. This command may be stop the worker, query its state or re-start
# it.
#
# This script requires sed and awk
#
# Author: Flavio Figueiredo - flaviovdf 'at' gmail.com
# Files to copy
TO_COPY=(spiderpig.jar spiderpig.sh spiderpigbg.sh lib/ lib-static/)
# Properties names to change
# TODO: We are not yet handling the control port, it is fixed per host
CONTROL_HOST_PROP="control\.hostname"
CONTROL_PORT_PROP="control\.port"
SERVICE_HOST_PROP="service\.hostname"
SERVICE_PORT_PROP="service\.port"
# Base directory
# TODO: Change script to have different base directories per worker
BASE_DIR="$HOME/.spiderpig/"
PROPERTIES_FNAME="worker.properties"
TMP_DIR="/tmp/.spiderpig-deployer"
PAK="spiderpig-worker.tar.gz"
# Exit codes
OK=0
ERROR=1
# Finds directory where running from
findself() {
SELF=`dirname $0`
}
# Prints usage
usage() {
echo "This script deploys spidepig workers to different host using ssh"
echo ""
echo "Usage: $SELF/$0 <base properties> <workers file> [option]"
echo ""
echo " <base properties>: a properties file with basic options to be"
echo " altered for each host"
echo ""
echo " <workers file>: a file with a host:port per line"
echo ""
echo " valid options"
echo " start: starts workers"
echo " stop: stops workers"
echo " restart: restarts workers"
echo " status: checks if workers are online"
echo " clean: clean worker folder for \$HOME/.spiderpig"
echo " deploy: create and copy worker package to \$HOME/.spiderpig"
echo " sshcheck: checks if host is online for ssh in port 22"
echo " kill: kill's java at the host (be carefull!)"
echo ""
echo "It is required to have ssh access without passwords and prompts."
echo "Create a key for this."
}
get_worker_dir() {
local host=$1
local port=$2
echo "$BASE_DIR/$host-$port"
}
get_tmp_dir() {
local host=$1
local port=$2
echo "$TMP_DIR/$host-$port"
}
ssh_check_one() {
local host=$1
ssh -n $host echo -n
echo $?
}
csshcheck() {
for host in ${!HOSTS[@]}; do
local state=`ssh_check_one $host 2> /dev/null`
if [ $state -eq $OK ]; then
echo "[sshcheck] Host $host is online for ssh"
else
echo "[sshcheck] Host $host is offline for ssh"
fi
done
}
cdeploy() {
declare -Al host_to_config
for host in ${!HOSTS[@]}; do
local port=${HOSTS[$host]}
local work_dir=`get_tmp_dir $host $port`
local ehost=`echo $host | sed 's/\./\\\\./g'`
local dep_dir=`get_worker_dir $host $port`
mkdir -p $work_dir > /dev/null 2>&1
#Creates new properties file
sed "s/$SERVICE_HOST_PROP\s*=.*/$SERVICE_HOST_PROP=$ehost/g" \
$BASE_PROPERTIES > $work_dir/$PROPERTIES_FNAME
sed -i "s/$SERVICE_PORT_PROP\s*=.*/$SERVICE_PORT_PROP=$port/g" \
$work_dir/$PROPERTIES_FNAME
sed -i "s/$CONTROL_HOST_PROP\s*=.*/$CONTROL_HOST_PROP=$ehost/g" \
$work_dir/$PROPERTIES_FNAME
local to_compact="$PROPERTIES_FNAME"
for fpath in ${TO_COPY[@]}; do
local to_compact="$SELF/$fpath $to_compact"
cp -r $SELF/$fpath $work_dir
done
cd $work_dir
echo "[deploy] Creating package for $host-$port"
tar zcf $PAK *
echo "[deploy] Creating $dep_dir on $host-$port"
ssh -n $host "mkdir -p $dep_dir 2> /dev/null"
echo "[deploy] Copying package for $host-$port"
scp $PAK $host:$dep_dir
echo "[deploy] Extracting package"
ssh -n $host "cd $dep_dir && tar zxf $PAK"
echo
cd - > /dev/null 2>&1
done
rm -rf $TMP_DIR 2> /dev/null
}
cclean() {
for host in ${!HOSTS[@]}; do
local port=${HOSTS[$host]}
local dir=`get_worker_dir $host $port`
echo "[clean] Cleaning $host-$port:$dir"
ssh -n $host "rm -rf $dir 2> /dev/null"
done
}
cstatus() {
for host in ${!HOSTS[@]}; do
local port=${HOSTS[$host]}
local dir=`get_worker_dir $host $port`
echo "[status] Checking worker state at $host-$port"
ssh -n $host \
"$dir/spiderpig.sh worker_status -c $dir/$PROPERTIES_FNAME"
done
}
cstop() {
for host in ${!HOSTS[@]}; do
local port=${HOSTS[$host]}
local dir=`get_worker_dir $host $port`
echo "[stop] Stopping worker at $host-$port"
ssh -n $host \
"$dir/spiderpig.sh worker_kill -c $dir/$PROPERTIES_FNAME"
done
}
cstart() {
for host in ${!HOSTS[@]}; do
local port=${HOSTS[$host]}
local dir=`get_worker_dir $host $port`
echo "[start] Starting worker at $host-$port"
ssh -n $host \
"$dir/spiderpigbg.sh worker_up -c $dir/$PROPERTIES_FNAME"
done
}
ckill() {
for host in ${!HOSTS[@]}; do
local port=${HOSTS[$host]}
local dir=`get_worker_dir $host $port`
echo "[kill] killing all java worker at $host-$port"
ssh -n $host "killall -9 java"
done
}
# Here we go!
main() {
BASE_PROPERTIES=$1
HOST_FILE=$2
OPTION=$3
if [ ! -e $BASE_PROPERTIES ]; then
echo "File $BASE_PROPERTIES does not exist" >&2
exit $ERROR
fi
if [ ! -e $HOST_FILE ]; then
echo "File $HOST_FILE does not exist" >&2
exit $ERROR
fi
# Awk magic. Basically splits hosts files and creates a string with
# each host separated by a single space. Does the same for the ports
local hosts_string=`awk -F':' '/^.*?:[0-9]+/ {print $1}' $HOST_FILE | \
paste -s -d' '`
local ports_string=`awk -F':' '/^.*?:[0-9]+/ {print $2}' $HOST_FILE | \
paste -s -d' '`
# This little trick creates arrays by splitting the string. IFS is the
# split char, read -a assigns. <<< tells which string to read
IFS=' ' read -a hostnames <<< $hosts_string
IFS=' ' read -a hostports <<< $ports_string
declare -Ag HOSTS=() # Global dict used everywhere from here
local n=$((${#hostnames[@]} - 1))
for i in `seq 0 $n`; do
HOSTS["${hostnames[i]}"]=${hostports[i]} # maps host to port
done
case $OPTION in
kill)
ckill
;;
start)
cstart
;;
stop)
cstop
;;
restart)
cstart
cstop
;;
status)
cstatus
;;
clean)
cclean
;;
deploy)
cdeploy
;;
sshcheck)
csshcheck
;;
*)
usage >&2
exit $ERROR
esac
}
findself
if [ $# -lt 3 ]; then
usage >&2
exit $ERROR
fi
main $*