Skip to content

Commit 022e73b

Browse files
committed
Bug# 33630199 : mysqldump make a non-consistent backup with
--single-transaction option [mysql-trunk] Problem: -------- Dumping data using mysqldump with --single-transaction and --set-gtid-purged=ON, while on the other session, INSERT queries are being sent, the dump file generated has inconsistent data. That is, the backup data has the gtids but not the corresponding data. Analysis: --------- When --single-tranasction is used, a transaction is started, which basically creates a snapshot of the data which is currently present in the database. After that, the function process_set_gtid_purged takes care of the gtids present in the system, fetching them, and adding the GTID_PURGED to the dump file. There's a short time window between these two things. If someone executes an INSERT statement or other queries during this fraction of second, the data won't be there in the dump since START TRANSACTION took a snapshot of the data before this moment, but the GTID set will contain these extra transactions. And the resultant dump file will have the old data with the new GTID set resulting in inconsistent dump. Fix: ---- 1) The fix is to use FTWRL (FLUSH TABLES WITH READ LOCK). Here, the point is that FTWRL allows us to both start a transaction with consistent snapshot, and consistently read the value of gtid_executed for that snapshot. 2) If the dump was started with --single-transaction, we would just call do_flush_tables_read_lock, this would acquire read locks on the tables. The FLUSH TABLES statement present inside of this function would also cause an increase in GTID by one. 3) Once locks have been acquired, just proceed normally. 4) The same bug existed in mysql-5.7, but in that case, mysqldump takes the GTID_EXECUTED when data dump is over, and the tables are unlocked, but in case of mysql-8.0, the GTID_EXECUTED is stored before the tables were unlocked, so in this case, the fix was just to perform a FTWRL before fetching the GTID_EXECUTED. Added new warning in the function : process_set_gtid_purged > Running mysqldump on a GTID enabled server, without --single-transaction or --master-data or --lock-all-tables will throw a warning about data dump might being inconsistent. Change-Id: If1e6f4b09ba398ade5e9bda20934db510b8871dc
1 parent e3713b4 commit 022e73b

File tree

6 files changed

+197
-5
lines changed

6 files changed

+197
-5
lines changed

client/mysqldump.cc

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5614,6 +5614,19 @@ static bool process_set_gtid_purged(MYSQL *mysql_con) {
56145614
"--all-databases --triggers --routines --events. \n");
56155615
}
56165616

5617+
if (!opt_single_transaction && !opt_lock_all_tables && !opt_master_data) {
5618+
fprintf(stderr,
5619+
"Warning: A dump from a server that has GTIDs "
5620+
"enabled will by default include the GTIDs "
5621+
"of all transactions, even those that were "
5622+
"executed during its extraction and might "
5623+
"not be represented in the dumped data. "
5624+
"This might result in an inconsistent data dump. \n"
5625+
"In order to ensure a consistent backup of the "
5626+
"database, pass --single-transaction or "
5627+
"--lock-all-tables or --master-data. \n");
5628+
}
5629+
56175630
set_session_binlog(false);
56185631
if (add_set_gtid_purged(mysql_con)) {
56195632
mysql_free_result(gtid_mode_res);
@@ -5894,17 +5907,16 @@ int main(int argc, char **argv) {
58945907

58955908
if (opt_slave_data && do_stop_slave_sql(mysql)) goto err;
58965909

5897-
if ((opt_lock_all_tables || opt_master_data ||
5898-
(opt_single_transaction && flush_logs)) &&
5910+
if ((opt_lock_all_tables || opt_master_data || opt_single_transaction) &&
58995911
do_flush_tables_read_lock(mysql))
59005912
goto err;
59015913

59025914
/*
59035915
Flush logs before starting transaction since
59045916
this causes implicit commit starting mysql-5.5.
59055917
*/
5906-
if (opt_lock_all_tables || opt_master_data ||
5907-
(opt_single_transaction && flush_logs) || opt_delete_master_logs) {
5918+
if (opt_lock_all_tables || opt_master_data || opt_single_transaction ||
5919+
opt_delete_master_logs) {
59085920
if (flush_logs || opt_delete_master_logs) {
59095921
if (mysql_refresh(mysql, REFRESH_LOG)) {
59105922
DB_error(mysql, "when doing refresh");
@@ -5921,6 +5933,7 @@ int main(int argc, char **argv) {
59215933
if (get_bin_log_name(mysql, bin_log_name, sizeof(bin_log_name))) goto err;
59225934
}
59235935

5936+
/* Start the transaction */
59245937
if (opt_single_transaction && start_transaction(mysql)) goto err;
59255938

59265939
/* Add 'STOP SLAVE to beginning of dump */

mysql-test/r/mysqldump_gtid.result

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ INSERT INTO t1 VALUES(3);
2020
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
2121
SET @MYSQLDUMP_TEMP_LOG_BIN = @@SESSION.SQL_LOG_BIN;
2222
SET @@SESSION.SQL_LOG_BIN= 0;
23-
/* SET @@GLOBAL.GTID_PURGED='+uuid:1-4';*/
23+
/* SET @@GLOBAL.GTID_PURGED='+uuid:1-5';*/
2424
DROP TABLE IF EXISTS `t1`;
2525
/*!40101 SET @saved_cs_client = @@character_set_client */;
2626
/*!50503 SET character_set_client = utf8mb4 */;
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
include/master-slave.inc
2+
Warnings:
3+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
4+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
5+
[connection master]
6+
CREATE DATABASE db;
7+
USE db;
8+
CREATE TABLE t(num INT, num2 INT);
9+
CREATE PROCEDURE insertParallely()
10+
BEGIN
11+
DECLARE counter INT DEFAULT 1;
12+
WHILE counter <= 10000 DO
13+
INSERT INTO db.t VALUES(counter, 1);
14+
SET counter = counter + 1;
15+
END WHILE;
16+
END$
17+
CALL insertParallely();;
18+
[connection server_1]
19+
[connection master]
20+
[connection slave]
21+
include/start_slave.inc
22+
[connection master]
23+
UPDATE db.t SET num2=2 WHERE num2=1;
24+
include/sync_slave_sql_with_master.inc
25+
[connection slave]
26+
include/assert.inc [The row count with num2=2 must be 10000 on slave]
27+
include/assert.inc [The slave's GTID_EXECUTED must be equal to the master's one]
28+
[connection master]
29+
DROP DATABASE db;
30+
include/rpl_end.inc
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
!include ../my.cnf
2+
3+
[mysqld]
4+
gtid-mode=on
5+
enforce-gtid-consistency
6+
server-id=100
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
--force-restart
Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
# ===== Purpose =====
2+
#
3+
# 1) The purpose of this test is to check if the GTIDs, and the data dumped
4+
# by mysqldump are consistent. Especially the case when the server has
5+
# GTIDs enabled, and there are transactions running in the background,
6+
# during the dump.
7+
# 2) We've to also make sure that the dump file can be restored on a
8+
# GTID aware server.
9+
#
10+
# ===== Implementation =====
11+
#
12+
# Consists of 2 steps
13+
#
14+
# - Step 1 -
15+
# 1) Create a test table 't', in database 'db'
16+
# 2) Create a stored procedure, which will be called just before mysqldump
17+
# is executed. This will help us to mimick a scenario in which there are
18+
# transactions going on in the background, while mysqldump is dumping
19+
# the databases.
20+
# 3) Call the procedure, then start mysqldump just after that
21+
#
22+
# - Step 2 -
23+
# 5) Now restore the dump file to the slave
24+
# 6) Execute some statements on the master
25+
# 7) Sync master, and slave
26+
# 8) The slave's GTID_EXECUTED must be equal to the master's one
27+
#
28+
# ===== Requirements =====
29+
#
30+
# 1) The dump captured by mysqldump with --single-transaction and
31+
# set-gtid-purged=ON must contain the exact set of transaction GTIDs
32+
# that were extracted.
33+
# 2) The dump file can be restored without any issue
34+
# 3) The replication is working fine
35+
# 4) The slave's GTID_EXECUTED must be equal to the master's one
36+
# when replication catches up.
37+
#
38+
# ===== References =====
39+
#
40+
# Bug#33630199 : mysqldump make a non-consistent backup with
41+
# --single-transaction option
42+
43+
--source include/have_binlog_format_row.inc
44+
--let $rpl_gtid_utils= 1
45+
--let $rpl_skip_start_slave= 1
46+
--source include/master-slave.inc
47+
48+
# Step 1
49+
# Running mysqldump, and inserting data into the tables at the same time
50+
51+
# File, which is going to be used by mysqldump
52+
let $mysqldumpfile = $MYSQLTEST_VARDIR/tmp/bug33630199_dump.sql;
53+
54+
# Create test db
55+
CREATE DATABASE db;
56+
57+
USE db;
58+
59+
# Create test table
60+
CREATE TABLE t(num INT, num2 INT);
61+
62+
# Create the stored procedure.
63+
# We need to keep on sending insert statements to the server,
64+
# while on the other session, mysqldump is dumping the data.
65+
DELIMITER $;
66+
67+
CREATE PROCEDURE insertParallely()
68+
BEGIN
69+
DECLARE counter INT DEFAULT 1;
70+
WHILE counter <= 10000 DO
71+
INSERT INTO db.t VALUES(counter, 1);
72+
SET counter = counter + 1;
73+
END WHILE;
74+
END$
75+
76+
DELIMITER ;$
77+
78+
# Start sending insert statements
79+
--send CALL insertParallely();
80+
81+
--let $rpl_connection_name= server_1
82+
--source include/rpl_connection.inc
83+
84+
# Dump on fast machines could be empty.
85+
# Make sure that somethings inserted.
86+
--let $wait_condition= SELECT COUNT(*) > 10 FROM db.t
87+
--source include/wait_condition.inc
88+
89+
# Start dump
90+
--exec $MYSQL_DUMP --compact --single-transaction --set-gtid-purged=ON -ER --databases db > $mysqldumpfile
91+
92+
--source include/rpl_connection_master.inc
93+
94+
--reap
95+
96+
# End of Step 1
97+
98+
# Step 2
99+
# Restoring the dump file, and syncing master-slave
100+
101+
--source include/rpl_connection_slave.inc
102+
103+
--exec $MYSQL_SLAVE --force < $mysqldumpfile
104+
--remove_file $mysqldumpfile
105+
106+
--source include/start_slave.inc
107+
108+
--source include/rpl_connection_master.inc
109+
110+
# Make sure that replication is working.
111+
# Execute an UPDATE that updates all the table data.
112+
# If some data is missing from the dump but not from the GTID set,
113+
# replication errors will occur.
114+
UPDATE db.t SET num2=2 WHERE num2=1;
115+
116+
--let $masters_gtid_executed=`SELECT @@GLOBAL.gtid_executed`
117+
118+
--source include/sync_slave_sql_with_master.inc
119+
120+
--source include/rpl_connection_slave.inc
121+
122+
# Conclusion:
123+
# Executed GTIDs on the master and the slave are equal
124+
125+
# Count the rows in 't' with num2=2 on slave, must be 10000
126+
--let $row_count = `SELECT 10000 = (SELECT COUNT(*) FROM db.t WHERE num2=2)`
127+
--let $assert_text= The row count with num2=2 must be 10000 on slave
128+
--let $assert_cond= $row_count
129+
--source include/assert.inc
130+
131+
# GTID_EXECUTED must be equal for both master, and slave
132+
--let $is_equal= `SELECT GTID_IS_EQUAL('$masters_gtid_executed', @@GLOBAL.gtid_executed)`
133+
--let $assert_text= The slave's GTID_EXECUTED must be equal to the master's one
134+
--let $assert_cond= $is_equal
135+
--source include/assert.inc
136+
137+
--source include/rpl_connection_master.inc
138+
DROP DATABASE db;
139+
140+
--source include/rpl_end.inc
141+
142+
# End of Step 2

0 commit comments

Comments
 (0)