-
Notifications
You must be signed in to change notification settings - Fork 0
/
redis-cluster
executable file
·320 lines (276 loc) · 12 KB
/
redis-cluster
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
#!/bin/bash
welcom () {
echo -e "\033[1;37m"
echo "-=# Redis Sharded Cluster Management #=-"
echo -e "\033[0m"
}
[ -f $(dirname $0)/redis-cluster.conf ] && source $(dirname $0)/redis-cluster.conf
[ -f $(dirname $0)/infra.conf ] && source $(dirname $0)/infra.conf
usage () {
echo "Usage: $(basename $0) <command>"
cat <<-EOT
Commands:
help display this help and exit
list show list of running redis nodes
create [ec2type] create main cluster node (with EC2 type if cpecified)
addnode [ec2type] add new redis node (with EC2 type if cpecified)
delnode <node_name> gracefully remove node from cluster and terminate it
status show Redis Sharded cluster status
EOT
}
die () {
echo -e "ERROR: $1"; echo; exit 1
}
check_params () {
param_list=(vpc_id aws_access_key_id aws_secret_access_key aws_region ec2type use_private_subnets key_name secgroup_name disk_size route53_zoneid route53_domain key_file bastion)
local errors=0
for i in "${param_list[@]}"; do
if [ -z "${!i}" ]; then echo "Please set parametr \"$i\""; errors=1; fi
done
if [ $errors -ne 0 ]; then die "Check your config file"; fi
export AWS_ACCESS_KEY_ID=$aws_access_key_id
export AWS_SECRET_ACCESS_KEY=$aws_secret_access_key
export AWS_DEFAULT_REGION=$aws_region
}
check_credentials () {
local vpc=`aws ec2 describe-vpcs`
[ $? -ne 0 ] && die "No access to AWS! Check credentials in config file"
[ -z `echo $vpc | jq ".[\"Vpcs\"]|.[]| select(.VpcId == \"$vpc_id\")|.VpcId"` ] && die "VPC with id \"$vpc_id\" not found! Check your config file"
[ -z `aws ec2 describe-key-pairs | jq ".[][]|select(.KeyName == \"$key_name\")|.KeyName"` ] && die "Key-pair with name \"$key_name\" not found! Check your config file"
[ -z `get_secgr_id` ] && die "Security group with name \"$secgroup_name\" not found! Check your config file"
}
check_utils () {
[ -z $(which jq) ] && die "Command-line JSON processor not found! Please install it by \033[1;37msudo apt-get install jq\033[0m"
[ -z $(which aws) ] && die "AWS Command Line Interface not found! Please install it by \033[1;37msudo pip install awscli\033[0m"
[ -z $(which column) ] && die "Columnate utility not found! Please unstall it by \033[1;37msudo apt-get install bsdmainutils\033[0m"
}
get_aws_data () {
aws_subnets=`aws ec2 describe-subnets --filters "[{\"Name\": \"vpc-id\", \"Values\": [\"$vpc_id\"]}, {\"Name\": \"state\", \"Values\": [\"available\"]}]"`
aws_instances=`aws ec2 describe-instances --filters "Name=vpc-id,Values=$vpc_id" | jq '.[] | .[] | .Instances[]'`
[ $(get_subnet) = "null" ] && die "There are no available subnets! Check your config file for option to use private subnets"
}
list_nodes () {
echo $aws_instances | jq 'select(.Tags) | {type: .InstanceType, id: .InstanceId, state: .State.Name, ip: .PrivateIpAddress, tags: .Tags[]} | select(.tags.Key == "Name") | {type, id, state, ip, name: .tags.Value}'
}
list_redis_nodes () {
echo $aws_instances | jq 'select(.Tags) | {type: .InstanceType, id: .InstanceId, state: .State.Name, ip: .PrivateIpAddress, tags: [.Tags[]]} | select(.tags[].Key == "redisnode") | select(.tags[].Key == "Name") | {type, id, state, ip, tags: .tags[]} | select(.tags.Key == "Name") | {type, id, state, ip, name: .tags.Value}'
}
subnet_count () {
if [ $use_private_subnets == 'yes' ]
then echo $aws_subnets | jq '[.["Subnets"]|.[]|select(.MapPublicIpOnLaunch == false)]|length'
else echo $aws_subnets | jq '[.["Subnets"]|.[]|select(.MapPublicIpOnLaunch == true) ]|length'
fi
}
ec2_count () {
list_nodes | jq '"\(.id) \(.name) \(.type) \(.state)"' | wc -l
}
# spread nodes over subnets
get_subnet () {
if [ $use_private_subnets == 'yes' ]
then echo $aws_subnets | jq "[.[\"Subnets\"]|.[]|select(.MapPublicIpOnLaunch == false)]| .[0].SubnetId" | tr -d '"'
else echo $aws_subnets | jq "[.[\"Subnets\"]|.[]|select(.MapPublicIpOnLaunch == true) ]| .[0].SubnetId" | tr -d '"'
fi
}
get_ami () {
curl -sSL http://cloud-images.ubuntu.com/query/trusty/server/released.current.txt \
| grep $aws_region | grep amd64 | grep hvm | grep -P "ebs\t" | awk '{print $8}'
}
get_node_names () {
list_nodes | jq '"\(.name)"' | tr -d '"'
}
get_node_name () {
names="$(get_node_names)"
local index
local name
index=0
while true; do
index=$(expr $index + 1)
name=$(printf "redis-cluster-%02i\n" $index)
case "${names[@]}" in *"$name"*) continue ;; esac
break
done
echo $name
}
get_secgr_id () {
aws ec2 describe-security-groups --filters Name=group-name,Values=$secgroup_name Name=vpc-id,Values=$vpc_id --query 'SecurityGroups[*].{Name:GroupId}' | jq '.[].Name' | tr -d '"'
}
launch () {
local newnode
local newnode_id
local newnode_name
local mainnode_ip
local ami
local type
type=$1
echo 'Get main node IP addr'
mainnode_ip=$(get_redis_ip)
[ -z "$mainnode_ip" ] && die "No main node found"
echo 'Get new node name'
newnode_name=$(get_node_name)
echo "Use node name \"$newnode_name\""
echo 'Get AMI image id'
ami=$(get_ami)
[ -z "$ami" ] && die "AMI image not found"
echo "Use AMI image \"$ami\" $(aws ec2 describe-images --image-ids $ami | jq '.[][].Name')"
echo 'Prepare user-data script'
sed \
-e "s;%region%;$aws_region;g" \
-e "s;%node_name%;$newnode_name;g" \
-e "s;%datadog_key%;$datadog_key;g" \
-e "s;%redismain%;$mainnode_ip;g" \
provision/node.sh.tmpl > provision/userdata
echo "Launch new node with EC2 type \"$type\""
newnode=$(aws ec2 run-instances \
--image-id $ami \
--key-name $key_name \
--security-group-ids $(get_secgr_id) \
--instance-type $type \
--subnet-id $(get_subnet) \
--block-device-mappings "[{\"DeviceName\":\"/dev/sda1\",\"Ebs\":{\"VolumeSize\":$disk_size,\"DeleteOnTermination\":true,\"VolumeType\":\"gp2\"}}]" \
--user-data file://provision/userdata)
rm -f provision/userdata
newnode_id=$(echo $newnode | jq '.Instances[].InstanceId' | tr -d '"')
[ -z "$newnode_id" ] && die "Node not lauched!"
echo "Node launched! Instance id is \"$newnode_id\", name is \"$newnode_name\""
aws ec2 create-tags --resources $newnode_id --tags Key=Name,Value=$newnode_name Key=redisnode,Value=true
echo 'New node labeled with tags'
echo 'Done'
echo
}
create () {
local newnode
local newnode_id
local newnode_ip
local newnode_name
local ami
local type
type=$1
newnode_name="redis-cluster-main"
newnode_id=`list_redis_nodes | jq "select(.name == \"$newnode_name\")| .id" | tr -d '"'`
if [ "$newnode_id" ]; then echo "Main node already running"; echo; return 1; fi
echo 'Get AMI image id'
ami=$(get_ami)
[ -z "$ami" ] && die "AMI image not found"
echo "Use AMI image \"$ami\" $(aws ec2 describe-images --image-ids $ami | jq '.[][].Name')"
echo 'Prepare user-data script'
sed \
-e "s;%region%;$aws_region;g" \
-e "s;%node_name%;$newnode_name;g" \
-e "s;%datadog_key%;$datadog_key;g" \
provision/mainnode.sh.tmpl > provision/userdata
echo "Launch main node with EC2 type \"$type\""
newnode=$(aws ec2 run-instances \
--image-id $ami \
--key-name $key_name \
--security-group-ids $(get_secgr_id) \
--instance-type $type \
--subnet-id $(get_subnet) \
--block-device-mappings "[{\"DeviceName\":\"/dev/sda1\",\"Ebs\":{\"VolumeSize\":$disk_size,\"DeleteOnTermination\":true,\"VolumeType\":\"gp2\"}}]" \
--user-data file://provision/userdata)
rm -f provision/userdata
newnode_id=$(echo $newnode | jq '.Instances[].InstanceId' | tr -d '"')
[ -z "$newnode_id" ] && die "Node not lauched!"
echo "Main node launched! Instance id is \"$newnode_id\", name is \"$newnode_name\""
aws ec2 create-tags --resources $newnode_id --tags Key=Name,Value=$newnode_name Key=redisnode,Value=true
echo 'New node labeled with tags'
echo "Add/Change DNS record for redis-cluster.$route53_domain"
newnode_ip=$(echo $newnode | jq '.Instances[].PrivateIpAddress' | tr -d '"')
echo 'Prepare and send Route53 request'
sed \
-e "s;%action%;UPSERT;g" \
-e "s;%ipaddr%;$newnode_ip;g" \
-e "s;%domain%;$route53_domain;g" \
provision/route53.json.tmpl > provision/route53.json
aws route53 change-resource-record-sets --hosted-zone-id $route53_zoneid --change-batch file://provision/route53.json >/dev/null
[ $? -ne 0 ] && die "Something wrong with DNS record update"
rm -f provision/route53.json
echo 'Done! Check status and wait while redis cluster appears'
echo
}
list_short () {
echo 'Listing current redis nodes'; echo
echo -e "\033[1;32mRUNNING NODES"
(echo 'ID NAME EC2TYPE STATE IP'; (list_redis_nodes | jq '"\(.id) \(.name) \(.type) \(.state) \(.ip)"' | tr -d '"') | sort -k 2) | column -t -e
echo -e "\033[0m"
echo 'Done'
}
initiate () {
echo 'Check utils installed'; check_utils
echo 'Check params in config file'; check_params
echo 'Check AWS credentials'; check_credentials
echo 'Load environment data from AWS'; get_aws_data
}
terminate () {
local node_id
local node_ip
local mainnode_ip
local answer
answer="no"
echo 'Get instance id'
node_id=`list_redis_nodes | jq "select(.name == \"$1\")| .id" | tr -d '"'`
if [ -z "$node_id" ]; then echo "Node $1 not found, use command \"list\" to see running nodes"; echo; return 1; fi
[ "$1" = "redis-cluster-main" ] && [ `list_redis_nodes | jq 'select(.name != "redis-cluster-main")| .id' | tr -d '"' | wc -l` -ne 0 ] && die "There are another cluster nodes! Please stop them before stop main node!"
echo -n "Ready to terminate node \"$1\" with id \"$node_id\", are you sure? type 'yes' to continue: "; read answer
if [ "$answer" == "yes" ]; then
if [ "$1" != "redis-cluster-main" ]; then
echo 'Gracefully remove node from Redis cluster'
mainnode_ip=$(get_redis_ip)
node_ip=`list_redis_nodes | jq "select(.name == \"$1\")| .ip" | tr -d '"'`
MAINNODE=$mainnode_ip REDISNODE=$node_ip BASTION_HOST=$bastion BASTION_KEY=$key_file ./provision/nodedel.sh
[ $? -ne 0 ] && die "Problem with gracefull removing from cluster"
fi
if [ "$1" = "redis-cluster-main" ]; then
echo "Delete DNS record redis-cluster.$route53_domain"
node_ip=`list_redis_nodes | jq "select(.name == \"$1\")| .ip" | tr -d '"'`
echo 'Prepare and send Route53 request'
sed \
-e "s;%action%;DELETE;g" \
-e "s;%ipaddr%;$node_ip;g" \
-e "s;%domain%;$route53_domain;g" \
provision/route53.json.tmpl > provision/route53.json
aws route53 change-resource-record-sets --hosted-zone-id $route53_zoneid --change-batch file://provision/route53.json >/dev/null
[ $? -ne 0 ] && echo "Something wrong with DNS record update"
rm -f provision/route53.json
fi
echo 'Terminating EC instance'
aws ec2 terminate-instances --instance-ids $node_id >/dev/null 2>&1
echo 'Done'
else
echo 'Ok, nothing to do'
fi
}
get_redis_ip () {
list_redis_nodes | jq 'select(.name == "redis-cluster-main")| .ip' | tr -d '"'
}
#
# main
#
welcom
case "$1" in
addnode)
initiate
if [ -z "$2" ]; then launch $ec2type; else launch $2; fi
;;
create)
initiate
if [ -z "$2" ]; then create $ec2type; else create $2; fi
;;
list)
initiate
list_short
;;
status)
initiate
REDISNODE=$(get_redis_ip) BASTION_HOST=$bastion BASTION_KEY=$key_file ./provision/status.sh
;;
help)
usage
;;
delnode)
if [ -z "$2" ]; then echo -e "Please specify node name: \033[1;37mredis-cluster rm <name>\033[0m"; echo; else initiate; terminate $2; fi
;;
*)
echo -e "Use \033[1;37m$(basename $0) help\033[0m to see usage info"
echo
;;
esac