Permalink
Browse files

Bug#20805298: BINLOG_ERROR_ACTION DOESN'T HANDLE SOME

FAILURES DURING BINLOG ROTATION

Analysis:
========
In case of hardware errors in binlog partition during binlog
rotate, we can see the following error being reported.

[ERROR] The server was unable to create a new log file. An
incident event has been written to the binary log which will
stop the slaves.
[ERROR] Can't generate a unique log-filename
master-bin.(1-999)

All replicas break either due to seeing incident event or
simply because the dump thread cannot access binlog
partition. binlog_error_action was developed for these
purposes (i,e; to avoid server accepting writes even though
binlog writes are failing), but it doesn't handle the above
case.

Fix:
===
An error handler has been added in such a way that, on
hardware failure during rotate, it will do the specific
action defined as part 'binlog_error_action' variable.
Also incident event will not be written to the binary log
in the above mentioned scenario.
  • Loading branch information...
Sujatha Sivakumar
Sujatha Sivakumar committed Aug 27, 2015
1 parent 3db5326 commit ffa4a65978901bc6fa9b41692e81a5f7d89342d3
@@ -84,12 +84,15 @@ eval SET GLOBAL debug=IF(LENGTH('$old_debug') > 0, CONCAT('$old_debug', ":d,erro
-- error ER_NO_UNIQUE_LOGFILE
FLUSH LOGS;
-- echo # assert: must show one binlog
-- source include/show_binary_logs.inc
--list_files $MYSQLD_DATADIR *master-bin.0*
### ACTION: clean up and move to next test
--disable_query_log
eval SET GLOBAL debug='$old_debug';
--enable_query_log
# Restart the server to enable binary log.
--let $rpl_server_number= 1
--source include/rpl_restart_server.inc
RESET MASTER;
-- echo ###################### TEST #3
@@ -145,6 +148,9 @@ SELECT count(*) FROM t2;
eval SET GLOBAL debug='$old_debug';
--enable_query_log
DELETE FROM t2;
# Restart the server to enable binary log.
--let $rpl_server_number= 1
--source include/rpl_restart_server.inc
RESET MASTER;
-- echo ###################### TEST #5
@@ -199,6 +205,9 @@ SET AUTOCOMMIT= 1;
eval SET GLOBAL debug='$old_debug';
--enable_query_log
DELETE FROM t2;
# Restart the server to enable binary log.
--let $rpl_server_number= 1
--source include/rpl_restart_server.inc
RESET MASTER;
-- echo ###################### TEST #7
@@ -218,15 +227,15 @@ SELECT count(*) FROM t4;
-- echo # assert: must show 1 entry
SELECT count(*) FROM t4;
-- echo ### check that the incident event is written to the current log
--disable_query_log
eval SET GLOBAL debug='$old_debug';
--enable_query_log
-- let $binlog_limit= 1
-- source include/show_binlog_events.inc
# clean up and move to next test
DELETE FROM t4;
# Restart the server to enable binary log.
--let $rpl_server_number= 1
--source include/rpl_restart_server.inc
RESET MASTER;
-- echo ###################### TEST #8
@@ -252,9 +261,26 @@ SELECT count(*) FROM t2;
-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
-- error ER_NO_UNIQUE_LOGFILE
-- eval LOAD DATA INFILE '$load_file' INTO TABLE t4
# Restart the server to enable binary log.
--let $rpl_server_number= 1
--source include/rpl_restart_server.inc
--disable_query_log
eval SET GLOBAL debug=IF(LENGTH('$old_debug') > 0, CONCAT('$old_debug', ":d,error_unique_log_filename"), 'd,error_unique_log_filename');
--enable_query_log
-- replace_result $MYSQLTEST_VARDIR MYSQLTEST_VARDIR
-- error ER_NO_UNIQUE_LOGFILE
-- eval LOAD DATA INFILE '$load_file' INTO TABLE t2
# Restart the server to enable binary log.
--let $rpl_server_number= 1
--source include/rpl_restart_server.inc
--disable_query_log
eval SET GLOBAL debug=IF(LENGTH('$old_debug') > 0, CONCAT('$old_debug', ":d,error_unique_log_filename"), 'd,error_unique_log_filename');
--enable_query_log
set @aaa = repeat('aaa',1000);
set @bbb = repeat('bbb',1000);
set @ccc = repeat('ccc',1000);
@@ -269,8 +295,30 @@ SELECT count(*) FROM t4;
-- echo # original rows remain
SELECT count(*) FROM t2;
# Restart the server to enable binary log.
--let $rpl_server_number= 1
--source include/rpl_restart_server.inc
--disable_query_log
eval SET GLOBAL debug=IF(LENGTH('$old_debug') > 0, CONCAT('$old_debug', ":d,error_unique_log_filename"), 'd,error_unique_log_filename');
--enable_query_log
SET @xxx = REPEAT('xxx', 1000);
SET @yyy = REPEAT('yyy', 1000);
-- error ER_NO_UNIQUE_LOGFILE
DELETE FROM t4 WHERE a IN (@xxx, @yyy) OR 1;
# Restart the server to enable binary log.
--let $rpl_server_number= 1
--source include/rpl_restart_server.inc
--disable_query_log
eval SET GLOBAL debug=IF(LENGTH('$old_debug') > 0, CONCAT('$old_debug', ":d,error_unique_log_filename"), 'd,error_unique_log_filename');
--enable_query_log
set @aaa = repeat('aaa',1000);
set @bbb = repeat('bbb',1000);
set @ccc = repeat('ccc',1000);
-- error ER_NO_UNIQUE_LOGFILE
DELETE FROM t2 WHERE a IN (@aaa, @bbb, @ccc) OR 1;
@@ -320,6 +368,9 @@ eval SET GLOBAL debug='$old_debug';
call mtr.add_suppression("MYSQL_BIN_LOG::open failed to sync the index file.");
call mtr.add_suppression("Could not open .*");
# Restart the server to enable binary log.
--let $rpl_server_number= 1
--source include/rpl_restart_server.inc
RESET MASTER;
SHOW WARNINGS;
@@ -477,6 +528,8 @@ call mtr.add_suppression("Can't generate a unique log-filename .*");
--disable_query_log
eval SET GLOBAL debug='$old_debug';
--enable_query_log
-- remove_file $MYSQLTEST_VARDIR/tmp/bug_46166.data
-- remove_file $MYSQLTEST_VARDIR/tmp/bug_46166-2.data
RESET SLAVE;
RESET MASTER;
--let $rpl_only_running_threads= 1
@@ -74,4 +74,21 @@ INSERT INTO t1 VALUES (2);
include/assert.inc [Count of elements in t1 should be 1.]
DROP table t1;
SET SESSION debug="-d,simulate_error_during_flush_cache_to_file";
"Test case10"
CREATE TABLE t1 (a INT) ENGINE=InnoDB;
SET SESSION debug='+d,error_unique_log_filename';
FLUSH LOGS;
ERROR HY000: Can't generate a unique log-filename master-bin.(1-999)
DROP TABLE t1;
SET SESSION debug="";
SHOW BINARY LOGS;
ERROR HY000: You are not using binary logging
"Test case11"
CREATE TABLE t1 (a INT) ENGINE=InnoDB;
SET GLOBAL binlog_error_action=ABORT_SERVER;
SET SESSION debug='+d,error_unique_log_filename';
FLUSH LOGS;
ERROR HY000: Binary logging not possible. Message: Either disk is full or file system is read only while rotating the binlog. Aborting the server
DROP TABLE t1;
SET SESSION debug="";
@@ -4,9 +4,6 @@ call mtr.add_suppression('MYSQL_BIN_LOG::open failed to sync the index file');
call mtr.add_suppression('Turning logging off for the whole duration of the MySQL server process.');
call mtr.add_suppression('Could not open .*');
call mtr.add_suppression('MYSQL_BIN_LOG::purge_logs failed to clean registers before purging logs.');
call mtr.add_suppression("The server was unable to create a new log file. "
"An incident event has been written to the binary "
"log which will stop the slaves.");
RESET MASTER;
flush logs;
flush logs;
@@ -1,9 +1,7 @@
call mtr.add_suppression("Next log extension: 2147483647. Remaining log filename extensions: 0.");
call mtr.add_suppression("Log filename extension number exhausted:");
call mtr.add_suppression("Can't generate a unique log-filename");
call mtr.add_suppression("The server was unable to create a new log file. "
"An incident event has been written to the binary "
"log which will stop the slaves.");
call mtr.add_suppression("Could not open .*");
RESET MASTER;
FLUSH LOGS;
ERROR HY000: Can't generate a unique log-filename master-bin.(1-999)
@@ -165,5 +165,50 @@ DROP table t1;
SET SESSION debug="-d,simulate_error_during_flush_cache_to_file";
--source include/restart_mysqld.inc
###############################################################################
# Bug#20805298: BINLOG_ERROR_ACTION DOESN'T HANDLE SOME
# FAILURES DURING BINLOG ROTATION
#
# Problem:
# ========
# Hardware errors in binlog partition during binlog rotate are not handled by
# binlog_error_action.
#
# Test:
# =====
# Simulate failure during creation of new binary log file name. Set
# binlog_error_action to "IGNORE_ERROR" and observe that the binary log gets
# disabled and the server continues by logging an appropriate error message in
# error log file. Set binlog_error_action to "ABORT_ERROR" and observe that
# the
# server aborts when creation of new binarylog file name fails.
###############################################################################
echo "Test case10";
# Test error scenario with binlog_error_action=IGNORE_ERROR
CREATE TABLE t1 (a INT) ENGINE=InnoDB;
SET SESSION debug='+d,error_unique_log_filename';
--error ER_NO_UNIQUE_LOGFILE
FLUSH LOGS;
DROP TABLE t1;
eval SET SESSION debug="$debug_save";
# Test to prove that binary log is disabled
--error ER_NO_BINARY_LOGGING
SHOW BINARY LOGS;
--source include/restart_mysqld.inc
echo "Test case11";
# Test error scenario with binlog_error_action=ABORT_SERVER
CREATE TABLE t1 (a INT) ENGINE=InnoDB;
SET GLOBAL binlog_error_action=ABORT_SERVER;
SET SESSION debug='+d,error_unique_log_filename';
--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
--error ER_BINLOG_LOGGING_IMPOSSIBLE
FLUSH LOGS;
--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
--enable_reconnect
--source include/wait_until_connected_again.inc
DROP TABLE t1;
# Cleanup
eval SET SESSION debug="$old";
@@ -14,9 +14,6 @@ call mtr.add_suppression('MYSQL_BIN_LOG::open failed to sync the index file');
call mtr.add_suppression('Turning logging off for the whole duration of the MySQL server process.');
call mtr.add_suppression('Could not open .*');
call mtr.add_suppression('MYSQL_BIN_LOG::purge_logs failed to clean registers before purging logs.');
call mtr.add_suppression("The server was unable to create a new log file. "
"An incident event has been written to the binary "
"log which will stop the slaves.");
let $old=`select @@debug`;
RESET MASTER;
@@ -22,9 +22,7 @@
call mtr.add_suppression("Next log extension: 2147483647. Remaining log filename extensions: 0.");
call mtr.add_suppression("Log filename extension number exhausted:");
call mtr.add_suppression("Can't generate a unique log-filename");
call mtr.add_suppression("The server was unable to create a new log file. "
"An incident event has been written to the binary "
"log which will stop the slaves.");
call mtr.add_suppression("Could not open .*");
-- source include/have_log_bin.inc
RESET MASTER;
@@ -23,9 +23,8 @@ FLUSH LOGS;
ERROR HY000: Can't generate a unique log-filename master-bin.(1-999)
# assert: must show one binlog
show binary logs;
Log_name File_size
master-bin.000001 #
master-bin.000001
include/rpl_restart_server.inc [server_number=1]
RESET MASTER;
###################### TEST #3
CREATE TABLE t1 (a INT);
@@ -53,6 +52,7 @@ SELECT count(*) FROM t2;
count(*)
1
DELETE FROM t2;
include/rpl_restart_server.inc [server_number=1]
RESET MASTER;
###################### TEST #5
LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/bug_46166-2.data' INTO TABLE t2;
@@ -80,6 +80,7 @@ count(*)
3
SET AUTOCOMMIT= 1;
DELETE FROM t2;
include/rpl_restart_server.inc [server_number=1]
RESET MASTER;
###################### TEST #7
SELECT count(*) FROM t4;
@@ -92,11 +93,8 @@ ERROR HY000: Can't generate a unique log-filename master-bin.(1-999)
SELECT count(*) FROM t4;
count(*)
1
### check that the incident event is written to the current log
include/show_binlog_events.inc
Log_name Pos Event_type Server_id End_log_pos Info
master-bin.000001 # Query # # BEGIN
DELETE FROM t4;
include/rpl_restart_server.inc [server_number=1]
RESET MASTER;
###################### TEST #8
SET @xxx = REPEAT('xxx', 1000);
@@ -113,9 +111,11 @@ count(*)
LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/bug_46166.data' INTO TABLE t4;
ERROR HY000: Can't generate a unique log-filename master-bin.(1-999)
include/rpl_restart_server.inc [server_number=1]
LOAD DATA INFILE 'MYSQLTEST_VARDIR/tmp/bug_46166.data' INTO TABLE t2;
ERROR HY000: Can't generate a unique log-filename master-bin.(1-999)
include/rpl_restart_server.inc [server_number=1]
set @aaa = repeat('aaa',1000);
set @bbb = repeat('bbb',1000);
set @ccc = repeat('ccc',1000);
@@ -133,9 +133,16 @@ count(*)
SELECT count(*) FROM t2;
count(*)
6
include/rpl_restart_server.inc [server_number=1]
SET @xxx = REPEAT('xxx', 1000);
SET @yyy = REPEAT('yyy', 1000);
DELETE FROM t4 WHERE a IN (@xxx, @yyy) OR 1;
ERROR HY000: Can't generate a unique log-filename master-bin.(1-999)
include/rpl_restart_server.inc [server_number=1]
set @aaa = repeat('aaa',1000);
set @bbb = repeat('bbb',1000);
set @ccc = repeat('ccc',1000);
DELETE FROM t2 WHERE a IN (@aaa, @bbb, @ccc) OR 1;
ERROR HY000: Can't generate a unique log-filename master-bin.(1-999)
@@ -173,6 +180,7 @@ SET SQL_LOG_BIN=1;
###################### TEST #10
call mtr.add_suppression("MYSQL_BIN_LOG::open failed to sync the index file.");
call mtr.add_suppression("Could not open .*");
include/rpl_restart_server.inc [server_number=1]
RESET MASTER;
SHOW WARNINGS;
Level Code Message
Oops, something went wrong.

0 comments on commit ffa4a65

Please sign in to comment.