Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
MDEV-27436: binlog corruption (/tmp no space left on device at the sa…
…me moment) This commit fixes several bugs in error handling around disk full when writing the statement/transaction binlog caches: 1. If the error occurs during a non-transactional statement, the code attempts to binlog the partially executed statement (as it cannot roll back). The stmt_cache->error was still set from the disk full error. This caused MYSQL_BIN_LOG::write_cache() to get an error while trying to read the cache to copy it to the binlog. This was then wrongly interpreted as a disk full error writing to the binlog file. As a result, a partial event group containing just a GTID event (no query or commit) was binlogged. Fixed by checking if an error is set in the statement cache, and if so binlog an INCIDENT event instead of a corrupt event group, as for other errors. 2. For LOAD DATA LOCAL INFILE, if a disk full error occured while writing to the statement cache, the code would attempt to abort and read-and-discard any remaining data sent by the client. The discard code would however continue trying to write data to the statement cache, and wrongly interpret another disk full error as end-of-file from the client. This left the client connection with extra data which corrupts the communication for the next command, as well as again causing an corrupt/incomplete event to be binlogged. Fixed by restoring the default read function before reading any remaining data from the client connection. Reviewed-by: Andrei Elkin <andrei.elkin@mariadb.com> Signed-off-by: Kristian Nielsen <knielsen@knielsen-hq.org>
- Loading branch information
Showing
8 changed files
with
286 additions
and
8 deletions.
There are no files selected for viewing
38 changes: 38 additions & 0 deletions
38
mysql-test/suite/rpl/r/rpl_binlog_cache_disk_full_loaddata.result
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
include/master-slave.inc | ||
[connection master] | ||
connection master; | ||
SET @save_binlog_stmt_cache_size= @@GLOBAL.binlog_stmt_cache_size; | ||
SET GLOBAL binlog_stmt_cache_size= 4096; | ||
CALL mtr.add_suppression('"No space left on device".*An incident event is written to binary log'); | ||
CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=MyISAM; | ||
FLUSH STATUS; | ||
SHOW STATUS LIKE "binlog_stmt_cache%"; | ||
Variable_name Value | ||
Binlog_stmt_cache_disk_use 0 | ||
Binlog_stmt_cache_use 0 | ||
SET @old_dbug= @@SESSION.debug_dbug; | ||
SET SESSION debug_dbug="+d,load_data_binlog_cache_error"; | ||
LOAD DATA CONCURRENT LOCAL INFILE 'std_data/bug30435_5k.txt' | ||
REPLACE INTO TABLE t1 (a); | ||
ERROR HY000: Error writing file '<FILENAME>' (Errcode: 28 "No space left on device") | ||
SET SESSION debug_dbug= @old_dbug; | ||
SHOW STATUS LIKE "binlog_stmt_cache%"; | ||
Variable_name Value | ||
Binlog_stmt_cache_disk_use 1 | ||
Binlog_stmt_cache_use 1 | ||
SELECT IF(COUNT(*) > 0 AND COUNT(*) < 5000, | ||
"ok", | ||
CONCAT("ERROR! Row count ", COUNT(*), " not as expected for partially executed query")) | ||
AS check_result | ||
FROM t1; | ||
check_result | ||
ok | ||
connection slave; | ||
include/wait_for_slave_sql_error_and_skip.inc [errno=1590] | ||
SELECT COUNT(*) FROM t1; | ||
COUNT(*) | ||
0 | ||
connection master; | ||
SET GLOBAL binlog_stmt_cache_size= @save_binlog_stmt_cache_size; | ||
DROP TABLE t1; | ||
include/rpl_end.inc |
51 changes: 51 additions & 0 deletions
51
mysql-test/suite/rpl/r/rpl_binlog_cache_disk_full_row.result
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
include/master-slave.inc | ||
[connection master] | ||
connection master; | ||
SET @save_binlog_stmt_cache_size= @@GLOBAL.binlog_stmt_cache_size; | ||
SET GLOBAL binlog_stmt_cache_size= 4096; | ||
CALL mtr.add_suppression('"No space left on device".*An incident event is written to binary log'); | ||
CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(255)) ENGINE=MyISAM; | ||
FLUSH STATUS; | ||
SHOW STATUS LIKE "binlog_stmt_cache%"; | ||
Variable_name Value | ||
Binlog_stmt_cache_disk_use 0 | ||
Binlog_stmt_cache_use 0 | ||
INSERT INTO t1 VALUES (0, CONCAT("?", "-", REPEAT("x", 200))); | ||
INSERT INTO t1 SELECT a+1, CONCAT(a, "-", REPEAT("x", 200)) FROM t1; | ||
INSERT INTO t1 SELECT a+2, CONCAT(a, "-", REPEAT("x", 200)) FROM t1; | ||
INSERT INTO t1 SELECT a+4, CONCAT(a, "-", REPEAT("x", 200)) FROM t1; | ||
INSERT INTO t1 SELECT a+8, CONCAT(a, "-", REPEAT("x", 200)) FROM t1; | ||
INSERT INTO t1 SELECT a+16, CONCAT(a, "-", REPEAT("x", 200)) FROM t1; | ||
INSERT INTO t1 SELECT a+32, CONCAT(a, "-", REPEAT("x", 200)) FROM t1; | ||
INSERT INTO t1 SELECT a+64, CONCAT(a, "-", REPEAT("x", 200)) FROM t1; | ||
INSERT INTO t1 SELECT a+128, CONCAT(a, "-", REPEAT("x", 200)) FROM t1; | ||
SHOW STATUS LIKE "binlog_stmt_cache%"; | ||
Variable_name Value | ||
Binlog_stmt_cache_disk_use 2 | ||
Binlog_stmt_cache_use 9 | ||
SET @old_dbug= @@SESSION.debug_dbug; | ||
SET SESSION debug_dbug="+d,simulate_disk_full_at_flush_pending"; | ||
INSERT INTO t1 SELECT a+256, CONCAT(a, "-", REPEAT("x", 200)) FROM t1; | ||
ERROR HY000: Error writing file '<FILENAME>' (Errcode: 28 "No space left on device") | ||
SET SESSION debug_dbug= @old_dbug; | ||
SHOW STATUS LIKE "binlog_stmt_cache%"; | ||
Variable_name Value | ||
Binlog_stmt_cache_disk_use 3 | ||
Binlog_stmt_cache_use 10 | ||
SELECT IF(COUNT(*) > 256 AND COUNT(*) < 512, | ||
"ok", | ||
CONCAT("ERROR! Row count ", COUNT(*), " not as expected for partially executed query")) | ||
AS check_result | ||
FROM t1; | ||
check_result | ||
ok | ||
ALTER TABLE t1 COMMENT '<mumble>'; | ||
connection slave; | ||
include/wait_for_slave_sql_error_and_skip.inc [errno=1590] | ||
SELECT COUNT(*) FROM t1; | ||
COUNT(*) | ||
256 | ||
connection master; | ||
SET GLOBAL binlog_stmt_cache_size= @save_binlog_stmt_cache_size; | ||
DROP TABLE t1; | ||
include/rpl_end.inc |
47 changes: 47 additions & 0 deletions
47
mysql-test/suite/rpl/t/rpl_binlog_cache_disk_full_loaddata.test
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
--source include/have_binlog_format_statement.inc | ||
--source include/have_debug.inc | ||
--source include/master-slave.inc | ||
|
||
--connection master | ||
# Set minimal cache size so smaller transaction can trigger spill to disk. | ||
SET @save_binlog_stmt_cache_size= @@GLOBAL.binlog_stmt_cache_size; | ||
SET GLOBAL binlog_stmt_cache_size= 4096; | ||
|
||
CALL mtr.add_suppression('"No space left on device".*An incident event is written to binary log'); | ||
CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=MyISAM; | ||
|
||
FLUSH STATUS; | ||
SHOW STATUS LIKE "binlog_stmt_cache%"; | ||
SET @old_dbug= @@SESSION.debug_dbug; | ||
SET SESSION debug_dbug="+d,load_data_binlog_cache_error"; | ||
--replace_regex /Error writing file '[^']+'/Error writing file '<FILENAME>'/ | ||
--error 3 | ||
LOAD DATA CONCURRENT LOCAL INFILE 'std_data/bug30435_5k.txt' | ||
REPLACE INTO TABLE t1 (a); | ||
SET SESSION debug_dbug= @old_dbug; | ||
SHOW STATUS LIKE "binlog_stmt_cache%"; | ||
# The actual number of rows left after the disk full error may change as | ||
# binlog event sizes are modified. So here we just test that we get partial | ||
# update from the last INSERT..SELECT that gets disk full error. | ||
SELECT IF(COUNT(*) > 0 AND COUNT(*) < 5000, | ||
"ok", | ||
CONCAT("ERROR! Row count ", COUNT(*), " not as expected for partially executed query")) | ||
AS check_result | ||
FROM t1; | ||
|
||
--save_master_pos | ||
|
||
--connection slave | ||
--let $slave_sql_errno= 1590 | ||
--source include/wait_for_slave_sql_error_and_skip.inc | ||
|
||
--sync_with_master | ||
SELECT COUNT(*) FROM t1; | ||
|
||
# Cleanup | ||
|
||
--connection master | ||
SET GLOBAL binlog_stmt_cache_size= @save_binlog_stmt_cache_size; | ||
DROP TABLE t1; | ||
|
||
--source include/rpl_end.inc |
61 changes: 61 additions & 0 deletions
61
mysql-test/suite/rpl/t/rpl_binlog_cache_disk_full_row.test
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
--source include/have_binlog_format_row.inc | ||
--source include/have_debug.inc | ||
--source include/master-slave.inc | ||
|
||
--connection master | ||
# Set minimal cache size so smaller transaction can trigger spill to disk. | ||
SET @save_binlog_stmt_cache_size= @@GLOBAL.binlog_stmt_cache_size; | ||
SET GLOBAL binlog_stmt_cache_size= 4096; | ||
|
||
CALL mtr.add_suppression('"No space left on device".*An incident event is written to binary log'); | ||
CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(255)) ENGINE=MyISAM; | ||
|
||
FLUSH STATUS; | ||
SHOW STATUS LIKE "binlog_stmt_cache%"; | ||
INSERT INTO t1 VALUES (0, CONCAT("?", "-", REPEAT("x", 200))); | ||
INSERT INTO t1 SELECT a+1, CONCAT(a, "-", REPEAT("x", 200)) FROM t1; | ||
INSERT INTO t1 SELECT a+2, CONCAT(a, "-", REPEAT("x", 200)) FROM t1; | ||
INSERT INTO t1 SELECT a+4, CONCAT(a, "-", REPEAT("x", 200)) FROM t1; | ||
INSERT INTO t1 SELECT a+8, CONCAT(a, "-", REPEAT("x", 200)) FROM t1; | ||
INSERT INTO t1 SELECT a+16, CONCAT(a, "-", REPEAT("x", 200)) FROM t1; | ||
INSERT INTO t1 SELECT a+32, CONCAT(a, "-", REPEAT("x", 200)) FROM t1; | ||
INSERT INTO t1 SELECT a+64, CONCAT(a, "-", REPEAT("x", 200)) FROM t1; | ||
INSERT INTO t1 SELECT a+128, CONCAT(a, "-", REPEAT("x", 200)) FROM t1; | ||
SHOW STATUS LIKE "binlog_stmt_cache%"; | ||
|
||
SET @old_dbug= @@SESSION.debug_dbug; | ||
SET SESSION debug_dbug="+d,simulate_disk_full_at_flush_pending"; | ||
--replace_regex /Error writing file '[^']+'/Error writing file '<FILENAME>'/ | ||
--error 3 | ||
INSERT INTO t1 SELECT a+256, CONCAT(a, "-", REPEAT("x", 200)) FROM t1; | ||
SET SESSION debug_dbug= @old_dbug; | ||
SHOW STATUS LIKE "binlog_stmt_cache%"; | ||
# The actual number of rows left after the disk full error may change as | ||
# binlog event sizes are modified. So here we just test that we get partial | ||
# update from the last INSERT..SELECT that gets disk full error. | ||
SELECT IF(COUNT(*) > 256 AND COUNT(*) < 512, | ||
"ok", | ||
CONCAT("ERROR! Row count ", COUNT(*), " not as expected for partially executed query")) | ||
AS check_result | ||
FROM t1; | ||
|
||
# A random extra event that helped show the bug that a partial event | ||
# group was binlogged. | ||
ALTER TABLE t1 COMMENT '<mumble>'; | ||
|
||
--save_master_pos | ||
|
||
--connection slave | ||
--let $slave_sql_errno= 1590 | ||
--source include/wait_for_slave_sql_error_and_skip.inc | ||
|
||
--sync_with_master | ||
SELECT COUNT(*) FROM t1; | ||
|
||
# Cleanup | ||
|
||
--connection master | ||
SET GLOBAL binlog_stmt_cache_size= @save_binlog_stmt_cache_size; | ||
DROP TABLE t1; | ||
|
||
--source include/rpl_end.inc |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters