Skip to content

Commit e60bd67

Browse files
santoshbandajtolmer
santoshbanda
authored and
jtolmer
committed
port streaming binlogs to 5.6.11
Summary: Currently 5.6 doesn't suppport compressed binlogs as input to mysqlbinlog. The original bug link is http://bugs.mysql.com/bug.php?id=49336 This fix allows mysqlbinlog to take multiple compressed files as direct input to mysqlbinlog rather than first decompressing the input files and passing them to mysqlbinlog. Ports the following change: 'Fix mysqlbinlog to take streaming file input' The following are the changes 1) Use read_log_event in check_header rather than my_b_seek if it is streaming file 2) After check_header, skip start_position - current_position bytes (rather than start_position bytes) Ports the following fix: 'Handle multiple files being streamed to mysqlbinlog' 5.6 takes multiple compressed file as input to mysqlbinlog. Changes in the original diff are 1) When multiple files are streamed, the binlog magic number occurs in the middle of stream and is causing problems. Skip such binlog magic magic numbers that occur in the middle of stream. Also, do not allow --stop-position with stream input because we do not know when the last file starts. Port the following fix: rpl.rpl_row_mysqlbinlog is using --stop-position with stdin so that rpl.rpl_row_mysqlbinlog passes. rpl.rpl_row_mysqlbinlog has 2 tests using --stop-position with stdin. Change them to expect error. Test Plan: ported the test cases. Reviewers: steaphan Reviewed By: steaphan
1 parent db9f9dd commit e60bd67

11 files changed

+1025
-78
lines changed

‎client/mysqlbinlog.cc‎

Lines changed: 95 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2590,14 +2590,18 @@ static Exit_status dump_remote_log_entries(PRINT_EVENT_INFO *print_event_info,
25902590
25912591
@param[in] logname Name of input binlog.
25922592
2593+
@param[in] stream_file Used to indicate that file is a stream and
2594+
therefore can't seek back and forth
2595+
25932596
@retval ERROR_STOP An error occurred - the program should terminate.
25942597
@retval OK_CONTINUE No error, the program should continue.
25952598
@retval OK_STOP No error, but the end of the specified range of
25962599
events to process has been reached and the program should terminate.
25972600
*/
25982601
static Exit_status check_header(IO_CACHE* file,
25992602
PRINT_EVENT_INFO *print_event_info,
2600-
const char* logname)
2603+
const char* logname,
2604+
bool stream_file)
26012605
{
26022606
DBUG_ENTER("check_header");
26032607
uchar header[BIN_LOG_HEADER_SIZE];
@@ -2614,14 +2618,22 @@ static Exit_status check_header(IO_CACHE* file,
26142618

26152619
pos= my_b_tell(file);
26162620

2617-
/* fstat the file to check if the file is a regular file. */
2618-
if (my_fstat(file->file, &my_file_stat, MYF(0)) == -1)
2621+
if (!stream_file)
2622+
{
2623+
/* fstat the file to check if the file is a regular file. */
2624+
if (my_fstat(file->file, &my_file_stat, MYF(0)) == -1)
2625+
{
2626+
error("Unable to stat the file.");
2627+
DBUG_RETURN(ERROR_STOP);
2628+
}
2629+
if ((my_file_stat.st_mode & S_IFMT) == S_IFREG)
2630+
my_b_seek(file, (my_off_t)0);
2631+
}
2632+
if (stream_file && pos != (my_off_t)0)
26192633
{
2620-
error("Unable to stat the file.");
2634+
error("Cannot rewind to header in a stream.");
26212635
DBUG_RETURN(ERROR_STOP);
26222636
}
2623-
if ((my_file_stat.st_mode & S_IFMT) == S_IFREG)
2624-
my_b_seek(file, (my_off_t)0);
26252637

26262638
if (my_b_read(file, header, sizeof(header)))
26272639
{
@@ -2634,6 +2646,72 @@ static Exit_status check_header(IO_CACHE* file,
26342646
DBUG_RETURN(ERROR_STOP);
26352647
}
26362648

2649+
/*
2650+
The rest of this function tries to figure out binlog format etc by reading
2651+
some events. We have two codepaths based on whether it is streaming file
2652+
or not. This is because we cannot go back and forth in a stream. Since the
2653+
streaming file only needs to be supported for 5.0+ formats, the code for
2654+
streaming path is simpler than the non-streaming case that handles all
2655+
formats.
2656+
*/
2657+
if (stream_file)
2658+
{
2659+
for (;;)
2660+
{
2661+
pos= my_b_tell(file);
2662+
2663+
if (pos >= start_position)
2664+
{
2665+
DBUG_RETURN(OK_CONTINUE);
2666+
}
2667+
2668+
Log_event *ev;
2669+
if (!(ev= Log_event::read_log_event(file, glob_description_event,
2670+
opt_verify_binlog_checksum)))
2671+
{
2672+
if (file->error)
2673+
{
2674+
error("Could not read a log_event at offset %llu;"
2675+
" this could be a log format error or read error.",
2676+
(ulonglong)pos);
2677+
DBUG_RETURN(ERROR_STOP);
2678+
}
2679+
// EOF
2680+
DBUG_RETURN(OK_CONTINUE);
2681+
}
2682+
2683+
if (ev->get_type_code() != FORMAT_DESCRIPTION_EVENT)
2684+
{
2685+
delete ev;
2686+
ev = NULL;
2687+
continue;
2688+
}
2689+
2690+
Format_description_log_event *new_description_event =
2691+
static_cast<Format_description_log_event *>(ev);
2692+
2693+
if (opt_base64_output_mode == BASE64_OUTPUT_AUTO)
2694+
{
2695+
/*
2696+
process_event will delete *description_event and set it to
2697+
the new one, so we should not do it ourselves in this
2698+
case.
2699+
*/
2700+
Exit_status retval= process_event(print_event_info,
2701+
new_description_event, pos,
2702+
logname);
2703+
if (retval != OK_CONTINUE)
2704+
DBUG_RETURN(retval);
2705+
}
2706+
else
2707+
{
2708+
delete glob_description_event;
2709+
glob_description_event= new_description_event;
2710+
}
2711+
}
2712+
DBUG_RETURN(OK_CONTINUE);
2713+
}
2714+
26372715
/*
26382716
Imagine we are running with --start-position=1000. We still need
26392717
to know the binlog format's. So we still need to find, if there is
@@ -2789,7 +2867,8 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info,
27892867
my_close(fd, MYF(MY_WME));
27902868
return ERROR_STOP;
27912869
}
2792-
if ((retval= check_header(file, print_event_info, logname)) != OK_CONTINUE)
2870+
if ((retval= check_header(file, print_event_info, logname, false))
2871+
!= OK_CONTINUE)
27932872
goto end;
27942873
}
27952874
else
@@ -2816,23 +2895,9 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info,
28162895
error("Failed to init IO cache.");
28172896
return ERROR_STOP;
28182897
}
2819-
if ((retval= check_header(file, print_event_info, logname)) != OK_CONTINUE)
2898+
if ((retval= check_header(file, print_event_info, logname, true))
2899+
!= OK_CONTINUE)
28202900
goto end;
2821-
if (start_position)
2822-
{
2823-
/* skip 'start_position' characters from stdin */
2824-
uchar buff[IO_SIZE];
2825-
my_off_t length,tmp;
2826-
for (length= start_position_mot ; length > 0 ; length-=tmp)
2827-
{
2828-
tmp= min<size_t>(length, sizeof(buff));
2829-
if (my_b_read(file, buff, (uint) tmp))
2830-
{
2831-
error("Failed reading from file.");
2832-
goto err;
2833-
}
2834-
}
2835-
}
28362901
}
28372902

28382903
if (!glob_description_event || !glob_description_event->is_valid())
@@ -3074,6 +3139,13 @@ int main(int argc, char** argv)
30743139
exit(1);
30753140
}
30763141

3142+
if ((argc == 1) && (stop_position != (ulonglong)(~(my_off_t)0)) &&
3143+
(!strcmp(argv[0], "-")))
3144+
{
3145+
error("stop_position not allowed when input is STDIN");
3146+
exit(1);
3147+
}
3148+
30773149
umask(((~my_umask) & 0666));
30783150
/* Check for argument conflicts and do any post-processing */
30793151
if (args_post_process() == ERROR_STOP)

‎mysql-test/extra/binlog_tests/mysqlbinlog_start_stop_streaming_input_1.inc‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# ==== Purpose ====
2+
#
3+
# Auxiliary file used by main.mysqlbinlog-start-stop-streaming-input.
4+
#
5+
# This file runs mysqlbinlog for a streaming file with a fixed set of parameters
6+
# specifying the source binlog, but varying how the start and stop positions/
7+
# datetime are specified.
8+
#
9+
# ==== Usage ====
10+
#
11+
# --let $start_position= <binlog offset>
12+
# --let $stop_position= <binlog offset>
13+
# --let $start_datetime= <timestamp>
14+
# --let $stop_datetime= <timestamp>
15+
# --let $file_options= <Streaming input files to mysqlbinlog>
16+
# --source extra/binlog_tests/mysqlbinlog_start_stop_1.inc
17+
#
18+
# Parameters:
19+
# $start_position
20+
# Offset to pass to --start-position
21+
# $stop_position
22+
# Offset to pass to --stop-position
23+
# $start_datetime
24+
# Offset to pass to --start-datetime
25+
# $stop_datetime
26+
# Offset to pass to --stop-datetime
27+
# $file_options
28+
# Specifying which input file to read.
29+
30+
--let $option= --base64-output=never
31+
--let $error= 0
32+
--source extra/binlog_tests/mysqlbinlog_start_stop_streaming_input_2.inc
33+
34+
--let $option= --offset=2
35+
--let $error= 0
36+
--source extra/binlog_tests/mysqlbinlog_start_stop_streaming_input_2.inc
37+
38+
--let $option= --start-position=$start_position
39+
--let $error= 0
40+
--source extra/binlog_tests/mysqlbinlog_start_stop_streaming_input_2.inc
41+
42+
--let $option= --stop-position=$stop_position
43+
--let $error= 1
44+
--source extra/binlog_tests/mysqlbinlog_start_stop_streaming_input_2.inc
45+
46+
--let $option= --start-position=$start_position --stop-position=$stop_position
47+
--let $error= 1
48+
--source extra/binlog_tests/mysqlbinlog_start_stop_streaming_input_2.inc
49+
50+
--let $option= "--start-datetime=$start_datetime"
51+
--let $error= 0
52+
--source extra/binlog_tests/mysqlbinlog_start_stop_streaming_input_2.inc
53+
54+
--let $option= "--stop-datetime=$stop_datetime"
55+
--let $error= 0
56+
--source extra/binlog_tests/mysqlbinlog_start_stop_streaming_input_2.inc
57+
58+
--let $option= "--start-datetime=$start_datetime" "--stop-datetime=$stop_datetime"
59+
--let $error= 0
60+
--source extra/binlog_tests/mysqlbinlog_start_stop_streaming_input_2.inc

‎mysql-test/extra/binlog_tests/mysqlbinlog_start_stop_streaming_input_2.inc‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# ==== Purpose ====
2+
#
3+
# Auxiliary file used by mysql-test/extra/binlog_tests/mysqlbinlog_start_stop_streaming_input_1.inc
4+
#
5+
# This prints a header, then runs mysqlbinlog once with given parameters.
6+
#
7+
# ==== Usage ====
8+
#
9+
# --let $options= X
10+
# --let $file_options= Y
11+
# --let $error= Z
12+
# --source include/mysqlbinlog-start-stop-streaming-input_2.inc
13+
#
14+
# Parameters:
15+
#
16+
# $options
17+
# Options that will be printed as a header in the test log.
18+
# $file_options
19+
# Streaming input files to mysqlbinlog
20+
# $error
21+
# Flag to determine if the execution expects an error or not
22+
23+
# Pretty-print $option_text
24+
--let $option_text= `SELECT SUBSTR(REPLACE('$option', '"', ''), 3)`
25+
while (`SELECT LOCATE('=', '$option_text')`)
26+
{
27+
--let $dash_pos= `SELECT LOCATE('--', '$option_text')`
28+
--let $option_text= `SELECT CONCAT(SUBSTR('$option_text', 1, LOCATE('=', '$option_text') - 1), IF($dash_pos, CONCAT(' ', SUBSTR('$option_text', $dash_pos + 2)), ''))`
29+
}
30+
--echo
31+
--echo ---- $option_text with streaming input----
32+
33+
# Print output
34+
--replace_regex /SET @@SESSION.GTID_NEXT= '.*'/SET @@SESSION.GTID_NEXT= 'GTID';/ /([0-9A-F\-]{36})\:[0-9]+\-[0-9]+/UUID:#-#/
35+
if ($error)
36+
{
37+
--error 1
38+
--exec cat $file_options | $MYSQL_BINLOG --short-form $option -
39+
}
40+
if (!$error)
41+
{
42+
--exec cat $file_options | $MYSQL_BINLOG --short-form $option -
43+
}

‎mysql-test/r/mysqlbinlog.result‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,44 @@ ROLLBACK /* added by mysqlbinlog */;
136136
/*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/;
137137
/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/;
138138

139+
--- Broken LOAD DATA with streaming input --
140+
/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/;
141+
/*!40019 SET @@session.max_insert_delayed_threads=0*/;
142+
/*!50003 SET @OLD_COMPLETION_TYPE=@@COMPLETION_TYPE,COMPLETION_TYPE=0*/;
143+
DELIMITER /*!*/;
144+
SET TIMESTAMP=1000000000/*!*/;
145+
SET @@session.pseudo_thread_id=999999999/*!*/;
146+
SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1/*!*/;
147+
SET @@session.sql_mode=1073741824/*!*/;
148+
SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/;
149+
/*!\C latin1 *//*!*/;
150+
SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/;
151+
SET @@session.lc_time_names=0/*!*/;
152+
SET @@session.collation_database=DEFAULT/*!*/;
153+
BEGIN
154+
/*!*/;
155+
use `test`/*!*/;
156+
SET TIMESTAMP=1000000000/*!*/;
157+
LOAD DATA LOCAL INFILE 'MYSQLTEST_VARDIR/tmp/SQL_LOAD_MB-#-#' INTO TABLE `t1` FIELDS TERMINATED BY '\t' ENCLOSED BY '' ESCAPED BY '\\' LINES TERMINATED BY '\n' (`word`)
158+
/*!*/;
159+
SET TIMESTAMP=1000000000/*!*/;
160+
COMMIT
161+
/*!*/;
162+
SET TIMESTAMP=1000000000/*!*/;
163+
BEGIN
164+
/*!*/;
165+
SET TIMESTAMP=1000000000/*!*/;
166+
insert into t1 values ("Alas")
167+
/*!*/;
168+
SET TIMESTAMP=1000000000/*!*/;
169+
COMMIT
170+
/*!*/;
171+
DELIMITER ;
172+
# End of log file
173+
ROLLBACK /* added by mysqlbinlog */;
174+
/*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/;
175+
/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/;
176+
139177
--- --database --
140178
/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/;
141179
/*!40019 SET @@session.max_insert_delayed_threads=0*/;
@@ -221,6 +259,35 @@ ROLLBACK /* added by mysqlbinlog */;
221259
/*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/;
222260
/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/;
223261

262+
--- --start-position with streaming input--
263+
/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/;
264+
/*!40019 SET @@session.max_insert_delayed_threads=0*/;
265+
/*!50003 SET @OLD_COMPLETION_TYPE=@@COMPLETION_TYPE,COMPLETION_TYPE=0*/;
266+
DELIMITER /*!*/;
267+
SET TIMESTAMP=1000000000/*!*/;
268+
SET @@session.pseudo_thread_id=999999999/*!*/;
269+
SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1/*!*/;
270+
SET @@session.sql_mode=1073741824/*!*/;
271+
SET @@session.auto_increment_increment=1, @@session.auto_increment_offset=1/*!*/;
272+
/*!\C latin1 *//*!*/;
273+
SET @@session.character_set_client=8,@@session.collation_connection=8,@@session.collation_server=8/*!*/;
274+
SET @@session.lc_time_names=0/*!*/;
275+
SET @@session.collation_database=DEFAULT/*!*/;
276+
BEGIN
277+
/*!*/;
278+
use `test`/*!*/;
279+
SET TIMESTAMP=1000000000/*!*/;
280+
insert into t1 values ("Alas")
281+
/*!*/;
282+
SET TIMESTAMP=1000000000/*!*/;
283+
COMMIT
284+
/*!*/;
285+
DELIMITER ;
286+
# End of log file
287+
ROLLBACK /* added by mysqlbinlog */;
288+
/*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/;
289+
/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/;
290+
224291
--- Remote --
225292
/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=1*/;
226293
/*!40019 SET @@session.max_insert_delayed_threads=0*/;

0 commit comments

Comments
 (0)