forked from Percona-QA/percona-qa
-
Notifications
You must be signed in to change notification settings - Fork 0
/
multirun_mysqld.sh
executable file
·196 lines (187 loc) · 10.7 KB
/
multirun_mysqld.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
#!/bin/bash
# Created by Ramesh Sivaraman, Percona LLC
# Updated by Roel Van de Paar, Percona LLC
# Start this script from within the base directory which contains ./bin/mysqld[-debug]
# User configurable variables
WORKDIR="/dev/shm" ## Working directory ("/dev/shm" preferred)
SQLFILE="./test.sql" ## SQL Input file
MYEXTRA="--no-defaults --event-scheduler=ON" ## MYEXTRA: Extra --options required for msyqld (may not be required)
MYEXTRA=" --no-defaults --plugin-load=tokudb=ha_tokudb.so --tokudb-check-jemalloc=0 --init-file=/home/roel/percona-qa/plugins_57.sql --binlog-group-commit-sync-delay=2047 "
SERVER_THREADS=(2 10 20 30 40 50) ## Number of server threads (x mysqld's). This is a sequence: (10 20) means: first 10, then 20 server if no crash was observed
CLIENT_THREADS=1 ## Number of client threads (y threads) which will execute the SQLFILE input file against each mysqld
AFTER_SHUTDOWN_DELAY=60 ## Wait this many seconds for mysqld to shutdown properly. If it does not shutdown within the allotted time, an error shows
# Internal variables
MYUSER=$(whoami)
MYPORT=$[20000 + $RANDOM % 9999 + 1]
DATADIR=`date +'%s'`
# Reference functions
echoit(){ echo "[$(date +'%T')] $1"; }
echoito(){ echo -ne "[$(date +'%T')] $1\r"; } # Used for on-screen updating of text
if [ ! -r ${SQL_FILE} ]; then
echoit "Assert: this script tried to read ${SQL_FILE} (as specified in the \"User configurable variables\" at the top of/inside the script), but it could not."
echoit "Please check if the file exists, if this script can read it, etc."
exit 1
fi
if [ ! -d ${WORKDIR} ]; then
echoit "Assert: {$WORKDIR} (as specified in the \"User configurable variables\" at the top of/inside the script) does not exist!"
exit 1
else # Workdir setup
WORKDIR="${WORKDIR}/$DATADIR"
mkdir -p ${WORKDIR}
if [ ! -d ${WORKDIR} ]; then
echoit "Assert: we tried to create ${WORKDIR}, but it does not exist after the creation attempt! Has this script write privileges there?"
exit 1
fi
fi
if [ -r ${PWD}/bin/mysqld ]; then
BIN=${PWD}/bin/mysqld
else
# Check if this is a debug build by checking if debug string is present in dirname
if [[ ${PWD} = *debug* ]]; then
if [ -r ${PWD}/bin/mysqld-debug ]; then
BIN=${PWD}/bin/mysqld-debug
else
echoit "Assert: there is no (script readable) mysqld binary at ${PWD}/bin/mysqld[-debug] ?"
exit 1
fi
else
echoit "Assert: there is no (script readable) mysqld binary at ${PWD}/bin/mysqld ?"
exit 1
fi
fi
echoit "Script work directory : ${WORKDIR}"
# Get version specific options
BIN=
if [ -r ${PWD}/bin/mysqld-debug ]; then BIN="${PWD}/bin/mysqld-debug"; fi # Needs to come first so it's overwritten in next line if both exist
if [ -r ${PWD}/bin/mysqld ]; then BIN="${PWD}/bin/mysqld"; fi
if [ "${BIN}" == "" ]; then echo "Assert: no mysqld or mysqld-debug binary was found!"; fi
MID=
if [ -r ${PWD}/scripts/mysql_install_db ]; then MID="${PWD}/scripts/mysql_install_db"; fi
if [ -r ${PWD}/bin/mysql_install_db ]; then MID="${PWD}/bin/mysql_install_db"; fi
START_OPT="--core-file" # Compatible with 5.6,5.7,8.0
INIT_OPT="--initialize-insecure" # Compatible with 5.7,8.0 (mysqld init)
INIT_TOOL="${BIN}" # Compatible with 5.7,8.0 (mysqld init), changed to MID later if version <=5.6
VERSION_INFO=$(${BIN} --version | grep -oe '[58]\.[01567]' | head -n1)
if [ "${VERSION_INFO}" == "5.1" -o "${VERSION_INFO}" == "5.5" -o "${VERSION_INFO}" == "5.6" ]; then
if [ "${MID}" == "" ]; then
echo "Assert: Version was detected as ${VERSION_INFO}, yet ./scripts/mysql_install_db nor ./bin/mysql_install_db is present!"
exit 1
fi
INIT_TOOL="${MID}"
INIT_OPT="--force --no-defaults"
START_OPT="--core"
elif [ "${VERSION_INFO}" != "5.7" -a "${VERSION_INFO}" != "8.0" ]; then
echo "WARNING: mysqld (${BIN}) version detection failed. This is likely caused by using this script with a non-supported distribution or version of mysqld. Please expand this script to handle (which shoud be easy to do). Even so, the scipt will now try and continue as-is, but this may fail."
fi
# Run SQL file from reducer<trial>.sh
for i in ${SERVER_THREADS[@]};do
# Start multiple mysqld service
SERVER_COUNT=0
MYSQLD=()
for j in `seq 1 ${i}`;do
SERVER_COUNT=$[ ${SERVER_COUNT} + 1 ];
echoito "Starting mysqld #${SERVER_COUNT}..."
MYPORT=$[ ${MYPORT} + 1 ]
mkdir ${WORKDIR}/${j} 2>/dev/null
$INIT_TOOL --no-defaults ${INIT_OPT} --basedir=${PWD} --datadir=${WORKDIR}/${j} > ${WORKDIR}/${j}_mysql_install_db.out 2>&1
CMD="bash -c \"set -o pipefail; ${BIN} ${MYEXTRA} ${START_OPT} --basedir=${PWD} --datadir=${WORKDIR}/${j} --port=${MYPORT}
--pid-file=${WORKDIR}/${j}_pid.pid --log-error=${WORKDIR}/${j}_error.log.out --socket=${WORKDIR}/${j}_socket.sock --user=${MYUSER}\""
eval $CMD > ${WORKDIR}/${j}_mysqld.out 2>&1 &
PIDV="$!"
MYSQLD+=(${PIDV})
echoit "Started mysqld #${SERVER_COUNT} with PID ${MYSQLD[j-1]}"
done
for j in `seq 1 ${i}`;do
x=0
${PWD}/bin/mysqladmin -uroot -S${WORKDIR}/${j}_socket.sock ping > /dev/null 2>&1
CHECK=$?
while [[ $CHECK != 0 ]]; do
${PWD}/bin/mysqladmin -uroot -S${WORKDIR}/${j}_socket.sock ping > /dev/null 2>&1
CHECK=$?
sleep 1
if [ $x == 60 ];then
echoit "[ERROR] Server not started: Check ${WORKDIR}/${j}_mysql_install_db.out, ${WORKDIR}/${j}_error.log.out and ${WORKDIR}/${j}_mysqld.out for more info"
exit 1;
fi
x=$[ $x+1 ]
done
done
# Start multiple mysql clients to test the SQL
MYSQLC=()
for j in `seq 1 ${i}`;do
echoit "Starting ${CLIENT_THREADS} client threads against mysqld #${j}..."
## The following line is for pquery testing (and remark entire for_do_done loop)
#$(cd `dirname $0` && pwd)/pquery/pquery --infile=${TRIAL}.out_out --database=test --threads=${CLIENT_THREADS} --user=root --socket=${WORKDIR}/${j}_socket.sock > ${WORKDIR}/${j}_pquery.out 2>&1 &
for (( thread=1; thread<=${CLIENT_THREADS}; thread++ )); do
${PWD}/bin/mysql -uroot --socket=${WORKDIR}/${j}_socket.sock -f < ${SQLFILE} > ${WORKDIR}/${j}_client-$thread.out 2>&1 &
PID="$!"
MYSQLC+=($PID)
done
# Check if mysqld process crashed immediately
if ! ${PWD}/bin/mysqladmin -uroot -S${WORKDIR}/${j}_socket.sock ping > /dev/null 2>&1; then
echoit "[!] Server crash/shutdown found : Check ${WORKDIR}/${j}_error.log.out for more info. Leaving state as-is and terminating. Consider using percona-qa/kill_all_procs.sh to cleanup after your research is done."
exit 1
fi
done
sleep 1 # Avoids last client not having started yet
# Check if mysql client finished
for k in "${MYSQLC[@]}"; do
while [[ ( -d /proc/$k ) && ( -z `grep zombie /proc/$k/status` ) ]]; do
# Check mysqld processes are still alive while waiting for client processes to finish
for j in `seq 1 ${i}`;do
if ! ${PWD}/bin/mysqladmin -uroot -S${WORKDIR}/${j}_socket.sock ping > /dev/null 2>&1; then
echoit "[!] Server crash/shutdown found: Check ${WORKDIR}/${j}_error.log.out for more info. Leaving state as-is and terminating. Consider using percona-qa/kill_all_procs.sh to cleanup after your research is done."
exit 1
fi
done
done
done
# Check mysqld processes are still alive after client processes are done
for j in `seq 1 ${i}`;do
if ! ${PWD}/bin/mysqladmin -uroot -S${WORKDIR}/${j}_socket.sock ping > /dev/null 2>&1; then
echoit "[!] Server crash/shutdown found: Check ${WORKDIR}/${j}_error.log.out for more info. Leaving state as-is and terminating. Consider using percona-qa/kill_all_procs.sh to cleanup after your research is done."
exit 1
fi
done
# Shutdown mysqld processes
for j in `seq 1 ${i}`;do
echoit "Shutting down mysqld #${j}..."
timeout --signal=9 ${AFTER_SHUTDOWN_DELAY}s ${PWD}/bin/mysqladmin -uroot -S${WORKDIR}/${j}_socket.sock shutdown > /dev/null 2>&1
if [ $? -eq 137 ]; then # Timeout was activated after ${AFTER_SHUTDOWN_DELAY} seconds, highly likely indicating a hang
echoit "[!] Potential server hang found: mysqld #{j} has not shutdown in ${AFTER_SHUTDOWN_DELAY} seconds. Check gdb --pid=${MYSQLD[j-1]}. Leaving state as-is and terminating. Consider using percona-qa/kill_all_procs.sh to cleanup after your research is done."
exit 1
fi
done
# Check for shutdown issues
for j in `seq 1 ${i}`;do
# Check for shutdown failure (mysqld still responding to mysqladmin pings)
PING=
timeout --signal=9 10s ${PWD}/bin/mysqladmin -uroot -S${WORKDIR}/${j}_socket.sock ping > /dev/null 2>&1
${PWD}/bin/mysqladmin -uroot -S${WORKDIR}/${j}_socket.sock ping
PING=$?
echo $PING
if [ $PING -eq 137 ]; then
echoit "[!] Potential server hang found: a mysqladmin ping to mysqld #${j} did not complete in 10 sconds. Check gdb --pid=${MYSQLD[j-1]} for more info. Leaving state as-is and terminating. Consider using percona-qa/kill_all_procs.sh to cleanup after your research is done."
exit 1
elif [ $PING -eq 0 ]; then
echoit "[!] Server hang found: mysqld #${j} has not shutdown in 60 seconds and is still responding to mysqladmin ping. Check gdb --pid=${MYSQLD[j-1]} for more info. Leaving state as-is and terminating. Consider using percona-qa/kill_all_procs.sh to cleanup after your research is done."
exit 1
elif [ $PING -ne 1 ]; then
echoit "[!] Unknown issue detected: a mysqladmin ping to mysqld #${j} returned exit status $PING, which is unkwnon to this script. Please research this code $PING and the current status of mysqld with gdb --pid=${MYSQLD[j-1]} for more info, then please update this script so it can handle this state. Leaving state as-is and terminating. Consider using percona-qa/kill_all_procs.sh to cleanup after your research is done."
exit 1
fi
# Check for core dump
if [ $(ls ${WORKDIR}/${j}/*core* 2>/dev/null | grep -vi "no such file or directory" | wc -l) -gt 0 ]; then
echoit "[!] Server crash found: mysqld #${j} has generated a core dump. Check ${WORKDIR}/${j}_error.log.out and $(ls -l ${WORKDIR}/${j}/*core* | tr '\n' ' ') for more info. Leaving state as-is and terminating. Consider using percona-qa/kill_all_procs.sh to cleanup after your research is done."
exit 1
fi
done
for j in `seq 1 ${i}`;do
echo "Checking PID ${MYSQLD[${i}]}"
done
kill -9 `printf '%s ' "${MYSQLD[@]}"` 2>/dev/null # For safety, though processes should be gone. Redirected stderr to /dev/null as otherwise 'multirun_mysqld.sh: line ___: kill: (_____) - No such process' errors would show.
if [ ${SERVER_THREADS[@]:(-1)} -ne ${i} ] ; then
echoit "Did not find server crash with ${i} mysqld processes. Restarting crash test with next set of mysqld processes."
rm -Rf ${WORKDIR}/*
fi
done