-
Notifications
You must be signed in to change notification settings - Fork 42
/
redis-tool
executable file
·1945 lines (1853 loc) · 59.3 KB
/
redis-tool
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#! /bin/bash
#
# redis cluster daily maintenance tool
#################################### global ###################################
readonly USAGE="Usage: $(basename $0) [OPTIONS] [command]
OPTIONS:
-h <IP>: Redis hostname (default: 127.0.0.1).
-p <port>: Redis port (default: 6379).
-i <nodefile>: Cluster nodes file. File format:
<slots1> <MasterAddr> [SlaveAddr] [SlaveAddr]
<slots2> <MasterAddr> [SlaveAddr] [SlaveAddr]
-a <password>: Password to use when connecting to redis.
You can also use the REDISCLI_AUTH environment
variable to pass this password more safely.
-c <count>: 1. Rolling times for command statistics (default is inf, used by 'moni').
2. Get slowlog number from each node (default: 100, used by 'slowlog').
3. Get key number from the node (default: 100000, used by 'keys').
-d <delay>: Waits <delay> seconds for monitor (default: 3, used by 'moni').
-t <time>: The running time(s) on trace (default: 10, used by 'trace').
-s: Process the specify node only (used by 'moni' & 'slowlog').
-l: Show average latency for TOP10 commands (used by 'moni').
-f <file>: 1. Monitor file name to load (used by 'trace').
2. Keys file name to statistics (used by 'keys').
-L <level>: Key prefix level (default: 3, usede by 'keys').
-H: Analyze the hot keys when use option '-f' (used by 'trace').
-C: Stat commands by cient host when use option '-f' (used by 'trace').
-r: Output raw data of cluster nodes (used by 'nodes').
-k <key>: Config name to get/set (used by 'config').
-v <value>: Config value to set (used by 'config').
-M: Only access masters config (used by 'config').
-S: Only access slaves config (used by 'config').
-w: Rewrite the config file after set online (used by 'config').
command:
nodes: Show cluster nodes status (default)
keys: Key prefix statistics
moni: Top10 command rolling statistics
trace: Hotspot key tracing statistics
slowlog: Querying cluster slow command logs
config: Get/Set cluster config
Examples:
# show cluster status by node(127.0.0.1:6379)
$ $(basename $0) nodes
# get cluster slowlog by node(127.0.0.1:7379)
$ $(basename $0) -p 7379 slowlog
# get config of loglevel by node(10.10.10.3:7379)
$ $(basename $0) -h 10.10.10.3 -p 7379 -k loglevel config\n"
# MYPID: pid of myself
readonly MYPID=$$
# XXX_PREFIX: temp file prefix name for subcommand
readonly CMDS_PREFIX=".RCTOOL-CMDS-"
readonly NODES_PREFIX=".RCTOOL-NODES-"
readonly INFOS_PREFIX=".RCTOOL-INFOS-"
readonly CLIENTS_PREFIX=".RCTOOL-CLIENTS-"
readonly MONI_PREFIX=".RCTOOL-MONI-"
readonly SLOWLOG_PREFIX=".RCTOOL-SLOWLOG-"
# redis server ip & password(default: 'REDISCLI_AUTH' environment variable)
REDIS_HOST="127.0.0.1"
REDIS_PORT=6379
REDIS_PASS="${REDISCLI_AUTH}"
# PREFIX_LVL: prefix key level, use option '-l' to set
PREFIX_LVL=3
# TRACE_TIME: how long to trace: default 10s
TRACE_TIME=10
# ROLL_TIME: rolling time for moni: default 3s
ROLL_TIME=3
# MONI_COUNT: moni N times: default infinite
MONI_COUNT=-1
# SLOW_COUNT: count of slowlog to get: default 100
SLOW_COUNT=100
# KEYS_COUNT: count of keys to stat by prefix: default 100000
KEYS_COUNT=100000
# CMD: subcommand: default 'nodes'
CMD="nodes"
# DATABASE: redis database: default 0
DATABASE=0
####################### function ########################
# clean temp files
function cleanup() {
rm -f ${CMDS_PREFIX}*.${MYPID} \
${NODES_PREFIX}*.${MYPID} \
${INFOS_PREFIX}*.${MYPID} \
${CLIENTS_PREFEX}*.${MYPID} \
${MONI_PREFIX}*.${MYPID} \
${SLOWLOG_PREFIX}*.${MYPID} || true
}
# when exception: kill self & subprocess
function exitup() {
cleanup
kill -9 0
}
# print info message
function debug() {
echo -e "$(date +'%F %T') \033[37m<INFO>\033[0m $@"
}
# print warning message
function warn() {
echo -e "$(date +'%F %T') \033[31m<WARN>\033[0m $@"
}
# check version
function version_ge() {
test "$(echo "$@" | tr " " "\n" | sort -rV | head -n 1)" == "$1"
}
#######################################
# check option is integer
# Globals:
# USAGE
# Arguments:
# 1 - option value
# 2 - option type
# Returns:
# None
#######################################
function assert_int() {
local -r int="${1:?missing value}"
if [[ ! "$int" =~ ^[0-9]+$ ]]; then
warn "Invalid integer for option '-${2}': $int"
echo -e "$USAGE"
exit 1
fi
}
#######################################
# check redis-cli to be useful
# Globals:
# REDISCLI_VERSION - redis-cli version
# Arguments:
# None
# Returns:
# None
#######################################
function check_redis_cli() {
# update PATH environment variable
export PATH=$REDIS_HOME:.:$PATH
local version=$(redis-cli -v 2>&1)
if [[ $? -ne 0 ]]; then
warn "Please use the REDIS_HOME environment variable to pass redis-cli path!"
exit 1
fi
REDISCLI_VERSION=$(echo $version | awk -F" |-" '{print $3}')
}
#######################################
# set REDISCLI_AUTH environment variable
# Globals:
# REDIS_PASS - redis password
# REDISCLI_AUTH - password used by redis-cli
# AUTH - password option
# Arguments:
# None
# Returns:
# None
#######################################
function set_rediscli_auth() {
if [[ -n "$REDIS_PASS" ]]; then
if version_ge "$REDISCLI_VERSION" "5.0.3"; then
export REDISCLI_AUTH="$REDIS_PASS"
else
AUTH="-a $REDIS_PASS"
fi
else
export -n REDISCLI_AUTH
AUTH=""
fi
}
#######################################
# cluster cluster nodes list
# Globals:
# REDIS_HOST - redis ip
# REDIS_PORT - redis port
# AUTH - password option
# CLUSTER_NODES_ALL - all nodes info
# MASTER_LIST - master addr list
# Arguments:
# None
# Returns:
# None
#######################################
function get_cluster_nodes() {
# ID ip:port flags masterID pingtime pongtime configepoch connect slots
local nodes_res=""
if [[ ${#CLUSTER_NODES_ALL} -eq 0 ]]; then
nodes_res=$(redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} ${AUTH} cluster nodes 2>&1)
if [[ $? -eq 0 && "${nodes_res:0:8}" != "LOADING " && "${nodes_res:0:7}" != "NOAUTH " && "${nodes_res:0:4}" != "ERR " ]]; then
# 1.masterID 2.role 3.ID 4.ip:port 5.flags 6.masterID 7.pingtime 8.pongtime 9.configepoch 10.connect 11.slots
CLUSTER_NODES_ALL=$(echo "$nodes_res"|awk '{if($4=="-"){print $1" master "$0}else{print $4" slave "$0}}'|sort)
elif [[ "$nodes_res" =~ "cluster support disabled" ]]; then
# redis isn't in cluster mode
local role_info=$(redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} ${AUTH} --raw role 2>&1)
local role=$(echo "$role_info"|head -1)
if [[ "$role" == "master" || "$role" == "slave" ]]; then
# used master role info: 1 master && >=1 slaves
if [[ "$role" == "slave" ]]; then
local master_ip=$(echo "$role_info"|head -2|tail -1)
local master_port=$(echo "$role_info"|head -3|tail -1)
local master_role_info=$(redis-cli -h ${master_ip} -p ${master_port} ${AUTH} --raw role 2>&1)
local master_role=$(echo "$master_role_info"|head -1)
if [[ "$master_role" == "master" ]]; then
role_info="$master_role_info"
REDIS_HOST=$master_ip
REDIS_PORT=$master_port
fi
fi
local awk_cmd='
{
idx+=1;
if(role==""){
role=$1;
if(role=="master") {
print myaddr,"master",myaddr,myaddr,"master","-","pingtime","pongtime","poch","connected","single-M";
}
} else if(role=="master") {
if(idx>2) {
if((idx-3)%3 == 0) {
slaveip=$1
} else if((idx-3)%3 == 1) {
slave=slaveip":"$1
print myaddr,"slave",slave,slave,"slave",myaddr,"pingtime","pongtime","poch","connected","single-S"
}
}
} else if(role=="slave") {
if(idx==2) {
masterip=$1;
} else if(idx==3) {
master=masterip":"$1
print master,"master",master,master,"master","-","pingtime","pongtime","poch","connected","single-M"
print master,"slave",myaddr,myaddr,"slave",master,"pingtime","pongtime","poch","connected","single-S"
}
}
}'
CLUSTER_NODES_ALL=$(echo "$role_info" | awk -v myaddr=${REDIS_HOST}:${REDIS_PORT} "$awk_cmd")
fi
fi
if [[ ${#CLUSTER_NODES_ALL} -eq 0 ]]; then
warn "Access redis(${REDIS_HOST}:${REDIS_PORT}) failed: \033[41;37m${nodes_res}\033[0m"
if [ "${nodes_res:0:7}" == "NOAUTH " ]; then
warn "Please input redis password by option '-a <password>'!"
elif [ "${nodes_res:0:8}" == "LOADING " ]; then
warn "Please retry the command after LOADING finished!"
else
warn "Please input active redis by option '-h <ip> -p <port>'!"
fi
exit 1
fi
fi
# cut cluster port from node addr
CLUSTER_NODES_ALL=$(echo "$CLUSTER_NODES_ALL"|sed -re 's/@[0-9]*//g')
# masters addr list
MASTER_LIST=$(echo "${CLUSTER_NODES_ALL}"|grep -w master|grep -v noaddr|awk '{print $4}')
# all nodes addr list
NODES_LIST=$(echo "${CLUSTER_NODES_ALL}"|grep -v noaddr|awk '{print $4}')
}
#######################################
# check redis status by ping
# Globals:
# REDIS_HOST - redis ip
# REDIS_PORT - redis port
# AUTH - password option
# Arguments:
# None
# Returns:
# None
#######################################
function check_redis_by_ping() {
local ping_res=$(redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} ${AUTH} ping 2>&1)
if [ "$ping_res" != "PONG" ]; then
warn "Access redis(${REDIS_HOST}:${REDIS_PORT}) failed: \033[41;37m${ping_res}\033[0m"
if [ "${ping_res:0:7}" == "NOAUTH " ]; then
warn "Please input redis password by option '-a <password>'!"
elif [ "${ping_res:0:8}" == "LOADING " ]; then
warn "Please retry the command after LOADING finished!"
else
warn "Please input active redis by option '-h <ip> -p <port>'!"
fi
exit 1
fi
}
#######################################
# print divide line with title
# Arguments:
# 1 - length of line
# 2 - character of line
# 3 - title
# 4 - new line
# Returns:
# divide line
#######################################
function draw_divide_line() {
local count=$1
local ch=$2
local title=$3
local newline=${4-'\n'}
local title_len=${#title}
local pre_len=$(((count-title_len)/2))
local str_result=""
for((i=1;i<=$pre_len;i++))
do
str_result+=$ch
done
str_result+="$title"
local sub_len=$((count-title_len-pre_len))
for((i=1;i<=$sub_len;i++))
do
str_result+=$ch
done
echo -en "$str_result$newline"
}
#######################################
# show progress bar
# Arguments:
# 1 - length of line
# 2 - character of line
# 3 - title
# 4 - new line
# Returns:
# progress bar
#######################################
function show_bar() {
local head_msg=$1
local total_num=$2
local proc_num=$3
local progress=0
# length of bar is 50
if [[ $total_num -eq 0 ]]; then
progress=50
else
progress=$((proc_num * 50 / total_num))
if [[ $progress -gt 50 ]]; then
progress=50
fi
fi
local progs_info=""
for ((i=0;i<$progress;i++)); do
progs_info="#${progs_info}"
done
if [ "$progs_info" = "" ]; then
progs_info="#"
fi
printf "<%s> | [%-50s]%d\n" $head_msg $progs_info $proc_num
}
#######################################
# show progress to figured out
# Arguments:
# 1 - nodes info
# Returns:
# progress bar
#######################################
function show_progress() {
local node_num=$(echo -e "$1"|wc -l)
local proc_num=0
# show progress with tty, and the cluster nodes > 20
tty >/dev/null
if [[ $? -eq 0 && $node_num -gt 20 ]]; then
while :;
do
proc_num=$(wc -l ${NODES_PREFIX}*.${MYPID} 2>/dev/null|tail -1|awk '{print $1}')
show_bar "TOTAL:$node_num" $node_num $proc_num
# break when figured out
if [[ $node_num -eq $proc_num ]]; then
break
fi
sleep 0.5
# reset cursor
echo -ne "\033[1A\r"
done
fi
}
#######################################
# config get/set for cluster
# Globals:
# CLUSTER_NODES_ALL - cluster nodes list
# REDIS_HOST - redis ip
# REDIS_PORT - redis port
# REDIS_PASS - redis password
# AUTH - password option
# CONFIG_SET - config set value
# CONFIG_KEY - config name to get/set
# CONFIG_VALUE - config value to set
# ROLE_TYPE - role type: master,slave,all
# Arguments:
# None
# Returns:
# get/set result
#######################################
function setget_config() {
# CLUSTER_NODES_ALL:
# 1.masterID 2.role 3.ID 4.ip:port 5.flags 6.masterID 7.pingtime 8.pongtime 9.configepoch 10.connect 11.slots
get_cluster_nodes
OLDIFS=$IFS
local head_line=$(draw_divide_line 80 "=")
local div_line=$(draw_divide_line 80 "-")
local conf_res=""
if [[ -z "$CONFIG_KEY" ]]; then
# audit config value with diffrent values
declare -A map_config
local conf_key=""
local conf_val=""
local node_count=$(echo "$CLUSTER_NODES_ALL"|grep "$ROLE_TYPE"|wc -l)
debug "Audit the config with diffrent value on \033[37m${ROLE_TYPE:-all}(count=$node_count)\033[0m nodes:"
local proc_count=0
for node in $(echo "$CLUSTER_NODES_ALL"|grep "$ROLE_TYPE"|grep -v noaddr|awk '{print $4}'|sort); do
# ip & port
local node_ip=${node%:*}
local node_port=${node##*:}
if [ "$node_ip" = "" ]; then
node_ip="127.0.0.1"
fi
# get all configs from redis
conf_res=$(redis-cli -h ${node_ip} -p ${node_port} ${AUTH} --raw config get "*"|awk '{if(x==0){x=1;printf("%s=",$1);}else{x=0;print $0}}')
if [[ ${#conf_res} -lt 200 ]]; then
warn "Get config from ${node_ip}:${node_port} failed: \033[41;37m${conf_res}\033[0m"
continue
fi
IFS=$'\n'
for conf in $(echo "$conf_res"); do
conf_key=${conf%%=*}
conf_val=${conf#*=}
map_config[$conf_key]="${map_config[$conf_key]}${conf_val}\n"
done
IFS=$OLDIFS
# show progress barcursor
if [[ $node_count -gt 20 ]]; then
proc_count=$((proc_count+1))
show_bar "TOTAL:$node_count" $node_count $proc_count
if [[ $node_count -gt $proc_count ]]; then
# reset cursor
echo -ne "\033[1A\r"
fi
fi
done
# audit the config values
head_line=$(draw_divide_line 100 "=")
div_line=$(draw_divide_line 100 "-")
local title=$(printf " %-32s| %s | %s" "CONFIG_NAME" "UNIQ" "DIFF_VALUES")
echo -e "${head_line}\n${title}\n${div_line}"
local diff_count=0
local diff_info=""
local info=""
for key in ${!map_config[@]}; do
diff_count=$(echo -en "${map_config[$key]}" | sort -u | wc -l)
local values=$(echo -en "${map_config[$key]}" | sort -u | tr '\n' ',')
# delete the last ','
values=${values%?}
if [[ ${#values} -gt 58 ]]; then
values="${values:0:55}..."
fi
# high light for diffrent values
info=$(printf " %-32s| %-4s | %-58s" "${key}" "$diff_count" "${values}")
if [[ $diff_count -gt 1 ]]; then
info="\033[43;31m${info}\033[0m"
fi
if [[ ${#diff_info} -gt 0 ]]; then
diff_info="${diff_info}\n${info}"
else
diff_info="${info}"
fi
done
# sort by UNIQ
echo -en "$diff_info" | sort -t '|' -nk 2,2
echo -e "${div_line}\n${title}\n${head_line}"
else
# get/set the config value
local title=$(printf " %-22s| %s" "REDIS" "$CONFIG_KEY")
local last_print=""
if [[ "$CONFIG_SET" = "yes" ]]; then
debug "SET the value of '$CONFIG_KEY' on \033[37m${ROLE_TYPE:-all}\033[0m nodes:"
# print at last for config set
last_print="${head_line}\n${title}\n${div_line}\n"
else
debug "GET the value of '$CONFIG_KEY' on \033[37m${ROLE_TYPE:-all}\033[0m nodes:"
# print immediatly for config get
echo -en "${head_line}\n${title}\n${div_line}\n"
fi
# Loop nodes to process
local old_pass="$REDIS_PASS"
for node in $(echo "$CLUSTER_NODES_ALL"|grep "$ROLE_TYPE"|grep -v noaddr|awk '{print $4}'|sort); do
# use the origin password
REDIS_PASS="$old_pass"
set_rediscli_auth
# ip & port
local node_ip=${node%:*}
local node_port=${node##*:}
if [ "$node_ip" = "" ]; then
node_ip="127.0.0.1"
fi
# CONFIG SET
if [[ "$CONFIG_SET" = "yes" ]]; then
conf_res=$(redis-cli -h ${node_ip} -p ${node_port} ${AUTH} config set "$CONFIG_KEY" "$CONFIG_VALUE")
# update the new password
if [[ "$CONFIG_KEY" = "requirepass" && "$conf_res" = "OK" ]]; then
REDIS_PASS="$CONFIG_VALUE"
set_rediscli_auth
fi
# CONFIG REWRITE
if [[ $WRITE_CONFIG = "yes" && "$conf_res" = "OK" ]]; then
conf_res="write config file: "$(redis-cli -h ${node_ip} -p ${node_port} ${AUTH} config rewrite 2>&1)
fi
debug "[${node_ip}:${node_port}]: CONFIG SET $CONFIG_KEY '$CONFIG_VALUE' : [$conf_res]"
fi
# CONFIG GET
conf_res=$(redis-cli -h ${node_ip} -p ${node_port} ${AUTH} --csv config get "$CONFIG_KEY" 2>&1 | awk -F',' '{print $2}')
conf_res=$(printf " %-22s| %s" "${node_ip}:${node_port}" "$conf_res")
if [[ "$CONFIG_SET" = "yes" ]]; then
# print at last for config set
last_print="${last_print}${conf_res}\n"
else
# print immediatly for config get
echo -e "$conf_res"
fi
done
if [[ "$CONFIG_SET" = "yes" ]]; then
# print the new value after config set
debug "Get the value of '$CONFIG_KEY' on \033[37m${ROLE_TYPE:-all}\033[0m nodes after CONFIG SET:"
echo -en "${last_print}"
fi
echo -e "${head_line}"
fi
}
#######################################
# get nodes info by ip
# Globals:
# CLUSTER_NODES_ALL - cluster nodes list
# REDIS_HOST - redis ip
# NODES_PREFIX - .RCTOOL-NODES-
# INFOS_PREFIX - .RCTOOL-INFOS-
# MYPID - pid
# Arguments:
# 1 - ip
# Returns:
# None
#######################################
function subprocess_nodes_by_ip() {
local deal_ip=$1
# init result file
local ret_file="${NODES_PREFIX}${deal_ip}.${MYPID}"
>$ret_file
local info_file="${INFOS_PREFIX}${deal_ip}.${MYPID}"
>$info_file
declare -A map_nodes
OLDIFS=$IFS
IFS=$'\n'
local master_addr=""
for node in $(echo "$CLUSTER_NODES_ALL"); do
IFS=$OLDIFS
# 1.masterID 2.role 3.ID 4.ip:port 5.flags 6.masterID 7.pingtime 8.pongtime 9.configepoch 10.connect 11.slots
local node_addr=$(echo "$node"|awk '{print $4}')
local no_addr=$(echo "$node"|grep -w noaddr|wc -l)
if [[ $no_addr -eq 1 ]]; then
node_addr="noaddr$node_addr"
fi
local role="slave"
local is_master=$(echo "$node"|grep -w master|wc -l)
if [[ $is_master -eq 1 ]]; then
master_addr="$node_addr"
role="master"
fi
local node_ip=${node_addr%:*}
local node_port=${node_addr##*:}
if [ "$deal_ip" != "" ]; then
if [ "$node_ip" != "$deal_ip" ]; then
continue
fi
else
if [ "$node_ip" != "noaddr" -a "$node_ip" != "" ]; then
continue
fi
fi
if [ "$node_ip" = "" ]; then
node_ip=${REDIS_HOST}
fi
# ELAPSED: time2 - time1
# $ echo -en "time\r\ntime\r\n"|redis-cli -h 172.16.17.3 -p 7290 --raw
# 1582680185
# 906676
# 1582680185
# 906940
local ustime1=0
local ustime2=0
local elapsed="-"
local status="FAIL"
local total_time=$(echo -en "time\r\ntime\r\n" | redis-cli -h ${node_ip} -p ${node_port} ${AUTH} --raw 2>/dev/null)
if [ "$total_time" != "" ]; then
if [ "${total_time:0:8}" == "LOADING " ];then
status="LOAD"
elif [ "${total_time:0:7}" == "NOAUTH " ];then
status="NOAUTH"
else
status="OK"
ustime1=$(echo $total_time|awk '{printf("%.f", $1*1000000+$2)}')
ustime2=$(echo $total_time|awk '{printf("%.f", $3*1000000+$4)}')
elapsed=$(expr $ustime2 - $ustime1)
fi
fi
# INFO
local node_info=""
local version="-"
local startime="-"
local usemem="-"
local usemem_byte="-"
local ops="-"
local clients="-"
local keynum="-"
# cpu
local ustime="-"
local cpu="-"
if [ "$status" == "OK" -o "$status" == "LOAD" ]; then
ustime=$(date +'%s %N'|awk '{printf("%.f", $1*1000000+$2/1000)}')
node_info=$(redis-cli -h ${node_ip} -p ${node_port} ${AUTH} info 2>/dev/null)
if [ "$node_info" != "" -a "${node_info:0:7}" != "NOAUTH " ]; then
version=$(echo "$node_info"|awk -F":|-|\r" '{if($1=="redis_version"){print $(NF-1)}}')
startime=$(echo "$node_info"|awk -F":|\r" '{if($1=="uptime_in_seconds"){print $2}}')
usemem=$(echo "$node_info"|awk -F":|\r" '{if($1=="used_memory_human"){print $2}}')
usemem_byte=$(echo "$node_info"|awk -F":|\r" '{if($1=="used_memory"){print $2}}')
ops=$(echo "$node_info"|awk -F":|\r" '{if($1=="instantaneous_ops_per_sec"){print $2}}')
clients=$(echo "$node_info"|awk -F":|\r" '{if($1=="connected_clients"){print $2}}')
# db0:keys=5001,expires=0,avg_ttl=0
keynum=$(echo "$node_info"|awk -F":|=|,|\r" '{if($1 ~ /^db[0-9]*/){keys+=$3}} END {print keys}')
if [ "$keynum" = "" ]; then
keynum="0"
fi
# cpu info
cpu=0
for N in $(echo "$node_info"|grep -E 'used_cpu_sys:|used_cpu_user:'|awk -F":" '{printf("%.f\n", $2*1000000)}');
do
cpu=$(($cpu + $N))
done
fi
fi
local node_id=$(echo "$node"|awk '{print $3}')
# 1.masteraddr 2.role 3.status 4.version 5.uptime 6.keys 7.usemem 8.client 9.ops 10.rtt 11.usemem_byte 12.ustime 13.cpu
map_nodes[$node_id]="$master_addr ${role} ${status} $version $startime $keynum $usemem $clients $ops $elapsed $usemem_byte $ustime $cpu "
echo "1" >>$ret_file
done
# sleep for a while
sleep 0.5
# calc the %cpu
IFS=$'\n'
for node in $(echo "$CLUSTER_NODES_ALL"); do
IFS=$OLDIFS
# 1.masterID 2.role 3.ID 4.ip:port 5.flags 6.masterID 7.pingtime 8.pongtime 9.configepoch 10.connect 11.slots
local node_addr=$(echo "$node"|awk '{print $4}')
local no_addr=$(echo "$node"|grep -w noaddr|wc -l)
if [[ $no_addr -eq 1 ]]; then
node_addr="noaddr$node_addr"
fi
local node_ip=${node_addr%:*}
local node_port=${node_addr##*:}
if [ "$deal_ip" != "" ]; then
if [ "$node_ip" != "$deal_ip" ]; then
continue
fi
else
if [ "$node_ip" != "noaddr" -a "$node_ip" != "" ]; then
continue
fi
fi
if [ "$node_ip" = "" ]; then
node_ip=${REDIS_HOST}
fi
# 1.masteraddr 2.role 3.status 4.version 5.uptime 6.keys 7.usemem 8.client 9.ops 10.rtt 11.usemem_byte 12.ustime 13.cpu 14.%cpu
local node_id=$(echo "$node"|awk '{print $3}')
local node_res="${map_nodes[$node_id]}"
local uslast=$(echo "$node_res"|awk '{print $12}')
local cpulast=$(echo "$node_res"|awk '{print $13}')
if [ "$cpulast" = "-" ];then
echo "$node_res - ${node}" >>$info_file
echo "1" >>$ret_file
continue
fi
local incrcpu=""
local ustime=$(date +'%s %N'|awk '{printf("%.f", $1*1000000+$2/1000)}')
local cpuinfo=$(redis-cli -h ${node_ip} -p ${node_port} ${AUTH} info cpu 2>/dev/null)
if [ "$cpuinfo" != "" -a "${cpuinfo:0:7}" != "NOAUTH " ]; then
local cpu=0
for N in $(echo "$cpuinfo"|grep -E 'used_cpu_sys:|used_cpu_user:'|awk -F":" '{printf("%.f\n", $2*1000000)}');
do
cpu=$(($cpu + $N))
done
incrcpu=$(echo "$cpu $cpulast $ustime $uslast"|awk '{if($1>=$2){printf("%.1f",($1-$2)*100/($3-$4)); }else{printf("%.1f",$1*100/($3-$4));}}')
else
incrcpu="-"
fi
echo "$node_res $incrcpu ${node}" >>$info_file
echo "1" >>$ret_file
done
IFS=$OLDIFS
}
#######################################
# show cluster nodes status
# Arguments:
# 1 - nodes info list
# Returns:
# cluster nodes status
#######################################
function show_cluster_status() {
# show cluster nodes
# SERVER STATUS ROLE VERSION UPTIME(s) KEYS USEMEM %CPU CLIENTS OPS RTT(us) SLOTS
awk_cmd='
BEGIN {
if(coltime <= 0) coltime=systime();
printf("================================================== %s ==================================================\n", strftime("%F %T", coltime));
printf("%-24s %-6s %-7s %-8s %-9s %-9s %-8s %-6s %-7s %-6s %-9s %-s \n",
"SERVER","STATUS","ROLE","VERSION","UPTIME(s)","KEYS","USEMEM","%CPU","CLIENTS","OPS","RTT(us)","SLOTS");
print "-------------------------------------------------------------------------------------------------------------------------"
}
{
mem=substr($8,1,length($8)-1);
unit=substr($8,length($8));
if(unit=="G") {
mem=mem*1073741824
} else if(unit=="M") {
mem=mem*1048576
} else if(unit=="K") {
mem=mem*1024
}
allkey+=$7;
allmem+=mem;
allcli+=$10;
allops+=$11;
allcpu+=$9;
allnode+=1;
if($4~/master/) {
masterkey+=$7;
mastermem+=mem;
mastercli+=$10;
masterops+=$11;
mastercpu+=$9;
masters+=1;
}
if($3=="OK") {
oknum+=1;
}else if($3=="FAIL") {
failnum+=1;
}else if($3=="NOAUTH") {
noauth+=1;
}else if($3=="LOAD") {
load+=1;
}
len=length($2);
if(len>21){
addr=substr($2,0,14)".."substr($2,len-4,5);
} else {
addr=$2;
}
match($2,/:[0-9]+$/);
masterip=substr($2,1,RSTART-1);
if($4~/master/) {
printf("%-24s ", addr);
mapip[masterip]+=1;
} else {
printf(" |-%-21s ", addr);
mapip[masterip]+=0;
}
printf("%-6s %-7s %-8s %-9s %-9s %-8s %-6s %-7s %-6s %-9s",$3,$4,$5,$6,$7,$8,$9,$10,$11,$12);
for(i=13;i<=NF;i++) {
printf(" %s",$i);
}
printf("\n");
} END {
print "_________________________________________________________________________________________________________________________"
if(allmem<1048576){
allmem=allmem/1024
allmem=sprintf("%.1fK",allmem)
}else if(allmem<1073741824){
allmem=allmem/1048576
allmem=sprintf("%.1fM",allmem)
}else{
allmem=allmem/1073741824
allmem=sprintf("%.1fG",allmem)
}
if(mastermem<1048576){
mastermem=mastermem/1024
mastermem=sprintf("%.1fK",mastermem)
}else if(mastermem<1073741824){
mastermem=mastermem/1048576
mastermem=sprintf("%.1fM",mastermem)
}else{
mastermem=mastermem/1073741824
mastermem=sprintf("%.1fG",mastermem)
}
printf("SUM(master/all): NODE=%s/%s KEY=%s/%s MEM=%s/%s CLI=%s/%s OPS=%s/%s CPU=%s/%s\n",
masters,allnode,masterkey,allkey,mastermem,allmem,mastercli,allcli,masterops,allops,mastercpu,allcpu);
if(noauth>0) {
printf("\033[41;37m!!!!! <WARNING> Access redis for \"NOAUTH\": Please use the option \"-a <password>\" !!!!!\033[0m\n");
}
if(load>0) {
printf("\033[43;37m!!!!! <WARNING> Please wait the status is \"LOAD\": The nodes are loading data !!!!!\033[0m\n");
}
if(failnum>0) {
printf("\033[41;37m!!!!! <WARNING> Please check the status is \"FAIL\": The nodes maybe offline !!!!!\033[0m\n");
}
minnum=masters
maxnum=0
for(ip in mapip) {
if(minnum > mapip[ip]) {
minnum=mapip[ip];
}
if(maxnum < mapip[ip]) {
maxnum=mapip[ip];
}
}
if(minnum != maxnum && maxnum > 1) {
printf("\033[41;37m!!!!! <WARNING> Masters unbalance on hosts: min/max = %s/%s !!!!!\033[0m\n", minnum,maxnum);
}
print "========================================================================================================================="
}'
# compact format
# 1,SERVER,STATUS,ROLE,VERSION,UPTIME,KEYS,USEMEM,%CPU,CLIENTS,OPS,DELAY,SLOTS&
echo "$1" | tr '&' '\n' | awk -F',' -v coltime=${2:-0} "$awk_cmd"
}
#######################################
# get cluster nodes status
# Globals:
# CLUSTER_NODES_ALL - cluster nodes list
# INFOS_PREFIX - .RCTOOL-INFOS-
# MYPID - pid
# OUTPUT_RAW_DATA - print raw data
# Arguments:
# 1 - nodes info list
# Returns:
# cluster nodes status
#######################################
function get_cluster_status() {
# CLUSTER_NODES_ALL:
# 1.masterID 2.role 3.ID 4.ip:port 5.flags 6.masterID 7.pingtime 8.pongtime 9.configepoch 10.connect 11.slots
get_cluster_nodes
# get all nodes info by ip
local last_ip="Nil"
for node in $(echo "$CLUSTER_NODES_ALL"|awk '{print $4}'|sort); do
# ip
local ip=${node%:*}
if [ "$last_ip" != "$ip" ]; then
last_ip=$ip
{
subprocess_nodes_by_ip $last_ip
}&
fi
done
# show progress: every node need to access twice in order to calc %cpu
show_progress "${CLUSTER_NODES_ALL}\n${CLUSTER_NODES_ALL}"
# wait subprocess done
wait
# 14: masteraddr role status version startime keynum usemem clients ops elapsed usemembyte ustime cpu %cpu
# 11: masterID role ID ip:port flags masterID pingtime pongtime configepoch connect slots
local nodes_res=$(cat ${INFOS_PREFIX}*.${MYPID} | sort)
# compact format
# 1,SERVER,STATUS,ROLE,VERSION,UPTIME,KEYS,USEMEM,CPU,CLIENTS,OPS,DELAY,SLOTS&
local awk_cmd='
{
printf("1,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s",$18,$3,$2,$4,$5,$6,$7,$14,$8,$9,$10);
if(NF > 24) {
for(i=25;i<=NF;i++) {
if(i==25) {
printf(",%s",$i);
} else {
printf(" %s",$i);
}
}
}
printf("&");
}'
local compres=$(echo "$nodes_res" | awk "$awk_cmd")
# delete the last '&'
compres=${compres:0:-1}
# echo "$compres"
if [ "$OUTPUT_RAW_DATA" = "yes" ]; then
echo "$compres"
else
show_cluster_status "$compres"
fi
}
#######################################
# get commands info by ip
# Globals:
# MASTER_LIST - cluster nodes list
# REDIS_HOST - redis ip
# MONI_PREFIX - .RCTOOL-MONI-
# MYPID - pid
# AUTH - auth info
# Arguments:
# 1 - ip
# Returns:
# commands info
#######################################
function subprocess_monitor_cmds_by_ip() {
local deal_ip=$1
# init result file
local ret_file="${MONI_PREFIX}${deal_ip}.${MYPID}"
>$ret_file
for node_addr in $(echo "$MASTER_LIST"|grep -w $deal_ip); do
# ADDR
local node_ip=${node_addr%:*}
local node_port=${node_addr##*:}
if [ "$node_ip" = "127.0.0.1" -o "$node_ip" = "" ]; then
node_ip=${REDIS_HOST}
fi
# cmdstat_get:calls=1,usec=1,usec_per_call=1.00
echo -en 'info stats\ninfo commandstats\ninfo cpu\n'|redis-cli -h ${node_ip} -p ${node_port} ${AUTH} 2>/dev/null >>$ret_file
if [ $? -ne 0 ]; then
warn "Can't connect to redis(${node_ip}:${node_port})!\n"
fi
done
# delete character '\r'
sed -i "s/\r//" ${ret_file}
}
#######################################
# rolling stats commands of masters
# Globals:
# MASTER_LIST - cluster nodes list
# REDIS_HOST - redis ip
# REDIS_PORT - redis port
# SINGLE_NODE - stat the specify node only
# CMDS_PREFIX - .RCTOOL-CMDS-
# MYPID - pid
# MONI_COUNT - run times
# MONI_LATENCY - show command average latency
# Arguments:
# None
# Returns:
# commands stats
#######################################
function moni_commands() {
# CLUSTER or SINGLE
if [ "$SINGLE_NODE" != "yes" ]; then
# MASTER_LIST
get_cluster_nodes
local node_num=$(echo "$MASTER_LIST"|wc -l)
debug "[CLUSTER] monitor with $node_num masters from ${REDIS_HOST}:${REDIS_PORT} every $ROLL_TIME seconds"
else
check_redis_by_ping
MASTER_LIST="${REDIS_HOST}:${REDIS_PORT}"
debug "[SINGLE] monitor only the node ${REDIS_HOST}:${REDIS_PORT} every $ROLL_TIME seconds"
fi
# cmdstat_get:calls=1,usec=1,usec_per_call=1.00
local awk_cmd='
{
if($1~/cmdstat_/) {
cmdname=substr($1,9);
if(lastcmd=="") {
lastcmd=cmdname;
}
if(lastcmd!=cmdname) {
printf("%s %s %s\n",lastcmd,cmdcalls,cmdusec);
lastcmd=cmdname;
cmdcalls=0;
cmdusec=0;
}
cmdcalls+=$3;
cmdusec+=$5;
}
} END {if(lastcmd!=""){printf("%s %s %s\n",lastcmd,cmdcalls,cmdusec);}}'
# rolling stats ...
OLDIFS=$IFS
local run_count=0
local base_cmd_stat=""
local last_cmd_stat=""
local last_time=0
local last_total=0
local last_cpu=0
local last_net_input=0