Skip to content

Commit a681699

Browse files
author
Nirbhay Choubey
committed
MDEV-10004: Galera's pc.recovery process fails in 10.1 with systemd
Galera recovery process works in two phases. In the first phase, mysqld is started as non-daemon with --wsrep-recover to recover and fetch the last logged global transaction ID. This ID is then used in second phase as the start position (--wsrep-start-position=XX) to start mysqld as daemon. As this process was implemented in mysqld_safe script, the recovery did not work when server was started using systemd. Fixed by introducing a shell script (wsrep_recovery.sh) that mimics the first phase of the recovery process.
1 parent 0645699 commit a681699

File tree

4 files changed

+152
-4
lines changed

4 files changed

+152
-4
lines changed

‎cmake/systemd.cmake

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,10 @@ MACRO(CHECK_SYSTEMD)
5555
IF(HAVE_SYSTEMD AND HAVE_SYSTEMD_SD_DAEMON_H AND HAVE_SYSTEMD_SD_LISTEN_FDS
5656
AND HAVE_SYSTEMD_SD_NOTIFY AND HAVE_SYSTEMD_SD_NOTIFYF)
5757
ADD_DEFINITIONS(-DHAVE_SYSTEMD)
58-
SET(SYSTEMD_SCRIPTS mariadb-service-convert galera_new_cluster)
58+
SET(SYSTEMD_SCRIPTS mariadb-service-convert galera_new_cluster galera_recovery)
5959
SET(SYSTEMD_DEB_FILES "usr/bin/mariadb-service-convert
6060
usr/bin/galera_new_cluster
61+
usr/bin/galera_recovery
6162
${INSTALL_SYSTEMD_UNITDIR}/mariadb.service
6263
${INSTALL_SYSTEMD_UNITDIR}/mariadb@.service
6364
${INSTALL_SYSTEMD_UNITDIR}/mariadb@bootstrap.service.d/use_galera_new_cluster.conf")

‎scripts/galera_recovery.sh

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#!/bin/sh
2+
3+
# Copyright (c) 2016 MariaDB Corporation
4+
#
5+
# This program is free software; you can redistribute it and/or modify
6+
# it under the terms of the GNU General Public License as published by
7+
# the Free Software Foundation; version 2 of the License.
8+
#
9+
# This program is distributed in the hope that it will be useful,
10+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
# GNU General Public License for more details.
13+
#
14+
# You should have received a copy of the GNU General Public License
15+
# along with this program; if not, write to the Free Software
16+
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
17+
18+
19+
# This script is intended to be executed by systemd. It starts mysqld with
20+
# --wsrep-recover to recover from a non-graceful shutdown, determines the
21+
# last stored global transaction ID and echoes it in --wsrep-start-position=XX
22+
# format. The output is then captured and used by systemd to start mysqld.
23+
# If the server was configured to start without wsrep, nothing is echoed.
24+
25+
cmdline_args=$@
26+
user="@MYSQLD_USER@"
27+
print_defaults="@bindir@/my_print_defaults"
28+
log_file=$(mktemp /tmp/wsrep_recovery.XXXXXX)
29+
euid=$(id -u)
30+
recovered_pos=""
31+
skipped=""
32+
start_pos=""
33+
start_pos_opt=""
34+
ret=0
35+
wsrep_on=0
36+
37+
log ()
38+
{
39+
local msg="$1"
40+
# Print all messages to stderr as we reserve stdout for printing
41+
# --wsrep-start-position=XXXX.
42+
echo "$msg" >&2
43+
}
44+
45+
finish()
46+
{
47+
rm -f "$log_file"
48+
}
49+
50+
trap finish EXIT
51+
52+
parse_arguments() {
53+
for arg do
54+
val=`echo "$arg" | sed -e "s;--[^=]*=;;"`
55+
case "$arg" in
56+
--wsrep[-_]on) wsrep_on=1 ;;
57+
--skip[-_]wsrep[-_]on) wsrep_on=0 ;;
58+
--wsrep[-_]on=*)
59+
if echo $val | grep -iq '\(ON\|1\)'; then
60+
wsrep_on=1
61+
else
62+
wsrep_on=0
63+
fi
64+
;;
65+
esac
66+
done
67+
}
68+
69+
wsrep_recover_position() {
70+
# Redirect server's error log to the log file.
71+
eval /usr/sbin/mysqld $cmdline_args --user=$user --wsrep_recover 2> "$log_file"
72+
ret=$?
73+
if [ $ret -ne 0 ]; then
74+
# Something went wrong, let us also print the error log so that it
75+
# shows up in systemctl status output as a hint to the user.
76+
log "WSREP: Failed to start mysqld for wsrep recovery: '`cat $log_file`'"
77+
exit 1
78+
fi
79+
80+
# Parse server's error log for recovered position. The server prints
81+
# "..skipping position recovery.." if started without wsrep.
82+
83+
recovered_pos="$(grep 'WSREP: Recovered position:' $log_file)"
84+
85+
if [ -z "$recovered_pos" ]; then
86+
skipped="$(grep WSREP $log_file | grep 'skipping position recovery')"
87+
if [ -z "$skipped" ]; then
88+
log "WSREP: Failed to recover position: '`cat $log_file`'"
89+
exit 1
90+
else
91+
log "WSREP: Position recovery skipped."
92+
fi
93+
else
94+
start_pos="$(echo $recovered_pos | sed 's/.*WSREP\:\ Recovered\ position://' \
95+
| sed 's/^[ \t]*//')"
96+
log "WSREP: Recovered position $start_pos"
97+
start_pos_opt="--wsrep_start_position=$start_pos"
98+
fi
99+
}
100+
101+
# Safety checks
102+
if [ -n "$log_file" -a -f "$log_file" ]; then
103+
[ "$euid" = "0" ] && chown $user $log_file
104+
chmod 600 $log_file
105+
else
106+
log "WSREP: mktemp failed"
107+
fi
108+
109+
parse_arguments `$print_defaults $cmdline_args --loose-verbose \
110+
mariadb mariadb_safe mysqld mysqld_safe safe_mysqld galera`
111+
112+
# Perform wsrep position recovery if wsrep_on=1, skip otherwise.
113+
if [ "$wsrep_on" -eq 1 ]; then
114+
wsrep_recover_position
115+
fi
116+
117+
echo "$start_pos_opt"
118+

‎support-files/mariadb.service.in

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,14 @@ CapabilityBoundingSet=CAP_IPC_LOCK
4848
# Execute pre and post scripts as root, otherwise it does it as User=
4949
PermissionsStartOnly=true
5050

51+
# Perform automatic wsrep recovery. When server is started without wsrep,
52+
# galera_recovery simply returns an empty string. In any case, however,
53+
# the script is not expected to return with a non-zero status.
54+
# It is always safe to unset _WSREP_START_POSITION environment variable.
55+
ExecStartPre=/bin/sh -c "systemctl unset-environment _WSREP_START_POSITION"
56+
ExecStartPre=/bin/sh -c "VAR=`/usr/bin/galera_recovery`; [ $? -eq 0 ] && \
57+
systemctl set-environment _WSREP_START_POSITION=$VAR || exit 1"
58+
5159
# Needed to create system tables etc.
5260
# ExecStartPre=/usr/bin/mysql_install_db -u mysql
5361

@@ -57,9 +65,12 @@ PermissionsStartOnly=true
5765
# This isn't a replacement for my.cnf.
5866
# _WSREP_NEW_CLUSTER is for the exclusive use of the script galera_new_cluster
5967
@SYSTEMD_EXECSTARTPRE@
60-
ExecStart=/usr/sbin/mysqld $MYSQLD_OPTS $_WSREP_NEW_CLUSTER
68+
ExecStart=/usr/sbin/mysqld $MYSQLD_OPTS $_WSREP_NEW_CLUSTER $_WSREP_START_POSITION
6169
@SYSTEMD_EXECSTARTPOST@
6270

71+
# Unset _WSREP_START_POSITION environment variable.
72+
ExecStartPost=/bin/sh -c "systemctl unset-environment _WSREP_START_POSITION"
73+
6374
KillMode=process
6475
KillSignal=SIGTERM
6576

‎support-files/mariadb@.service.in

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,21 @@ CapabilityBoundingSet=CAP_IPC_LOCK
5555
# Execute pre and post scripts as root, otherwise it does it as User=
5656
PermissionsStartOnly=true
5757

58+
# Perform automatic wsrep recovery. When server is started without wsrep,
59+
# galera_recovery simply returns an empty string. In any case, however,
60+
# the script is not expected to return with a non-zero status.
61+
# It is always safe to unset _WSREP_START_POSITION%I environment variable.
62+
ExecStartPre=/bin/sh -c "systemctl unset-environment _WSREP_START_POSITION%I"
63+
ExecStartPre=/bin/sh -c "VAR=`/usr/bin/galera_recovery \
64+
--defaults-file=@INSTALL_SYSCONF2DIR@/my%I.cnf`; [ $? -eq 0 ] && \
65+
systemctl set-environment _WSREP_START_POSITION%I=$VAR || exit 1"
66+
# Alternate: (remove ConditionPathExists above)
67+
# use [mysqld.INSTANCENAME] as sections in my.cnf
68+
#
69+
#ExecStartPre=/bin/sh -c "VAR=`/usr/bin/galera_recovery \
70+
# --defaults-group-suffix=%I`; [ $? -eq 0 ] && \
71+
# systemctl set-environment _WSREP_START_POSITION%I=$VAR || exit 1"
72+
5873
# Needed to create system tables etc.
5974
# ExecStartPre=/usr/bin/mysql_install_db -u mysql
6075

@@ -67,12 +82,15 @@ PermissionsStartOnly=true
6782
# Note: Place $MYSQLD_OPTS at the very end for its options to take precedence.
6883

6984
ExecStart=/usr/sbin/mysqld --defaults-file=@INSTALL_SYSCONF2DIR@/my%I.cnf \
70-
$_WSREP_NEW_CLUSTER $MYSQLD_OPTS
85+
$_WSREP_NEW_CLUSTER $_WSREP_START_POSITION%I $MYSQLD_OPTS
7186
# Alternate: (remove ConditionPathExists above)
7287
# use [mysqld.INSTANCENAME] as sections in my.cnf
7388
#
7489
# ExecStart=/usr/sbin/mysqld --defaults-group-suffix=%I \
75-
# $_WSREP_NEW_CLUSTER $MYSQLD_OPTS
90+
# $_WSREP_NEW_CLUSTER $_WSREP_START_POSITION%I $MYSQLD_OPTS
91+
92+
# Unset _WSREP_START_POSITION environment variable.
93+
ExecStartPost=/bin/sh -c "systemctl unset-environment _WSREP_START_POSITION%I"
7694

7795
KillMode=process
7896
KillSignal=SIGTERM

0 commit comments

Comments
 (0)